Changeset 1490

Show
Ignore:
Timestamp:
04/08/08 23:21:56 (5 months ago)
Author:
kindlund
Message:

Finished initial round of integration with Manager. Still have to fix disk checking.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • honeyclient/branches/exp/kindlund-simpler_agent/bin/StartManager.pl

    r1413 r1490  
    11#!perl -Ilib 
    2  
    3 # $Id$ 
     2####################################################################### 
     3# Created on:  Apr 08, 2008 
     4# File:        StartManager.pl 
     5# Description: Start up script for manager-based operations. 
     6# 
     7# CVS: $Id$ 
     8# 
     9# @author knwang, kindlund 
     10# 
     11# Copyright (C) 2007-2008 The MITRE Corporation.  All rights reserved. 
     12# 
     13# This program is free software; you can redistribute it and/or 
     14# modify it under the terms of the GNU General Public License 
     15# as published by the Free Software Foundation, using version 2 
     16# of the License. 
     17#  
     18# This program is distributed in the hope that it will be useful, 
     19# but WITHOUT ANY WARRANTY; without even the implied warranty of 
     20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
     21# GNU General Public License for more details. 
     22#  
     23# You should have received a copy of the GNU General Public License 
     24# along with this program; if not, write to the Free Software 
     25# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 
     26# 02110-1301, USA. 
     27# 
     28####################################################################### 
     29 
     30BEGIN { 
     31    our $VERSION = 1.02; 
     32} 
     33our ($VERSION); 
     34 
     35=pod 
     36 
     37=head1 NAME 
     38 
     39StartManager.pl - Perl script to start the Manager on the 
     40host system. 
     41 
     42=head1 SYNOPSIS 
     43 
     44 StartManager.pl [options] [http://www.google.com http://www.cnn.com ...] 
     45 
     46 Options: 
     47 --help               This help message. 
     48 --man                Print full man page. 
     49 --driver_name=       Name of driver to use. 
     50 --master_vm_config=  Absolute path to the master VM configuration to use. 
     51 --url_list=          File containing newline separated URLs to use. 
     52 
     53=head1 OPTIONS 
     54 
     55=over 4 
     56 
     57=item B<--help> 
     58 
     59Print a brief help message and exits. 
     60 
     61=item B<--driver_name=> 
     62 
     63Specifies the driver name to use.  If none is specified, the 
     64default will be used. 
     65 
     66=item B<--master_vm_config=> 
     67 
     68Specifies the master VM configuration file to use.  If none 
     69is specified, the default will be used. 
     70 
     71=item B<--url_list=> 
     72 
     73If specified, the newline separated URLs inside this file will 
     74be parsed and fed into the Manager upon startup. 
     75 
     76=back 
     77 
     78=head1 DESCRIPTION 
     79 
     80This program starts the Manager on the host system.  If URLs 
     81are specified on the command-line, the program will  
     82assign a base priority to each URL and feed them into the Manager 
     83for additional processing. 
     84 
     85This program will run until manually terminated by the user, by 
     86pressing CTRL-C. 
     87 
     88=head1 SEE ALSO 
     89 
     90L<http://www.honeyclient.org/trac> 
     91 
     92=head1 REPORTING BUGS 
     93 
     94L<http://www.honeyclient.org/trac/newticket> 
     95 
     96=head1 AUTHORS 
     97 
     98Darien Kindlund, E<lt>kindlund@mitre.orgE<gt> 
     99 
     100Kathy Wang, E<lt>knwang@mitre.orgE<gt> 
     101 
     102=head1 COPYRIGHT & LICENSE 
     103 
     104Copyright (C) 2007-2008 The MITRE Corporation.  All rights reserved. 
     105 
     106This program is free software; you can redistribute it and/or 
     107modify it under the terms of the GNU General Public License 
     108as published by the Free Software Foundation, using version 2 
     109of the License. 
     110  
     111This program is distributed in the hope that it will be useful, 
     112but WITHOUT ANY WARRANTY; without even the implied warranty of 
     113MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
     114GNU General Public License for more details. 
     115  
     116You should have received a copy of the GNU General Public License 
     117along with this program; if not, write to the Free Software 
     118Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 
     11902110-1301, USA. 
     120 
     121=cut 
    4122 
    5123use strict; 
    …… 
    7125use Carp (); 
    8126 
     127# Include Pod Library 
     128use Pod::Usage; 
     129 
    9130# Include Dumper Library 
    10131use Data::Dumper; 
    …… 
    17138 
    18139# Include Getopt Parser 
    19 use Getopt::Long; 
     140use Getopt::Long qw(:config auto_help ignore_case_always); 
    20141 
    21142# Include utility access to global configuration. 
    22143use HoneyClient::Util::Config qw(getVar); 
    23144 
    24 # Include Manager Library 
    25 use HoneyClient::Manager; 
    26145 
    27146# Include Logging Library 
    …… 
    34153# Namely, the initial set of URLs that they want the Agent to use. 
    35154 
    36 # Change to 'HoneyClient::Agent::Driver::Browser::IE' or 
    37 #           'HoneyClient::Agent::Driver::Browser::FF' 
    38 my $driver = undef; 
    39 my $config = undef; 
    40 my $maxrel = undef; 
    41 my $nexturl = ""; 
    42 my $urllist= ""; 
    43  
    44 # TODO: Need --help option, along with sanity checking. 
    45 # TODO: Also need a decent POD for this code. 
    46 GetOptions('driver=s'             => \$driver, 
    47            'master_vm_config=s'   => \$config, 
    48            'url_list=s'           => \$urllist, 
    49            'max_relative_links:i' => \$maxrel); 
    50  
    51 # Sanity Check.  Make sure $driver is set. 
    52 unless (defined($driver)) { 
    53     $driver = getVar(name      => "default_driver", 
    54                      namespace => "HoneyClient::Agent"); 
    55 } 
    56  
    57 # Sanity Check.  Make sure $max_relative_links is set. 
    58 unless (defined($maxrel)) { 
    59     $maxrel = getVar(name      => "max_relative_links_to_visit", 
    60                      namespace => "HoneyClient::Agent::Driver::Browser"); 
    61 } 
     155# Inputs. 
     156my $driver_name = undef; 
     157my $master_vm_config = undef; 
     158my $url_list= ""; 
     159 
     160GetOptions('driver_name=s'        => \$driver_name, 
     161           'master_vm_config=s'   => \$master_vm_config, 
     162           'url_list=s'           => \$url_list, 
     163           'man'                  => sub { pod2usage(-exitstatus => 0, -verbose => 2) }, 
     164           'version'              => sub { 
     165                                        print "MITRE HoneyClient Project (http://www.honeyclient.org)\n" . 
     166                                              "------------------------------------------------------\n" . 
     167                                              $0  . " (v" . $VERSION . ")\n"; 
     168                                        exit(0); 
     169                                     }) or pod2usage(2); 
    62170 
    63171# Go through the list of urls to create the array 
    …… 
    66174my @urls; 
    67175push( @urls, @ARGV );  
    68 if( -e $urllist ){ 
    69     open URL, $urllist; 
     176if( -e $url_list ){ 
     177    open URL, $url_list; 
    70178    push(@urls, <URL>); 
    71179} 
    72180 
    73 # Get the first url from the list 
    74 # Create a hashtable in the form: url => 1 for links_to_visit  
    75 chomp @urls; 
    76 my $firsturl = shift @urls; 
    77 my %remaining_urls; 
     181# Get the base priority. 
     182my $priority = getVar(name      => "command_line_base_priority", 
     183                      namespace => "HoneyClient::Manager"); 
     184 
     185# Create a hashtable in the form: url => priority. 
     186my $work = {}; 
    78187foreach(@urls){ 
    79188    # We assign our initial list of URLs a priority of 1000, so that 
    80189    # they'll be (likely to be) selected first, before going to any other 
    81190    # external URLs found from subsequent drive operations. 
    82     $remaining_urls{$_} = 1000; 
     191    chomp; 
     192    if ($_ ne "") { 
     193        $work->{$_} = $priority; 
     194    } 
    83195} 
    84196 
    85 my $agentState = HoneyClient::Manager->run( 
    86                     driver           => $driver, 
    87                     master_vm_config => $config, 
    88                     agent_state      => encode_base64(nfreeze({ 
    89                         $driver => { 
    90                             next_link_to_visit => $firsturl, 
    91                             max_relative_links_to_visit => $maxrel, 
    92                             links_to_visit => \%remaining_urls, 
    93                          }, 
    94                     })),  
    95                  ); 
    96  
     197# Start the Manager. 
     198require HoneyClient::Manager; 
     199HoneyClient::Manager->run( 
     200    driver_name      => $driver_name, 
     201    master_vm_config => $master_vm_config, 
     202    work             => $work, 
     203); 

  • honeyclient/branches/exp/kindlund-simpler_agent/etc/honeyclient.xml

    r1461 r1490  
    179179    </Crawler> 
    180180    <Manager> 
     181        <!-- TODO: Is this still needed? --> 
    181182        <manager_state description="Upon termination, the Manager will attempt to append a complete copy of its state into this file, if specified." default=""> 
    182183            manager.dump 
    …… 
    192193            8089 
    193194        </port> 
    194         <max_agent_error_count description="When the Agent is running, this value is the maximum number of SOAP communication errors the Manager will ignore (e.g., timeouts) before the Manager suspends the corresponding VM and clones a new Agent.  These errors mainly occur when the Manager loses connectivity to the Agent for some reason (i.e., software/OS crashing inside the VM).  Otherwise, the Manager would loop forever trying to reconnect with the faulty VM.  This value should never be set to 0 and should always be positive." default="3"> 
    195             3 
    196         </max_agent_error_count> 
     195        <num_urls_to_process description="If database support is enabled, this value indicates how many URLs the Manager will fetch from the database, process, and report back, before fetching for more URLs." default="10"> 
     196            10 
     197        </num_urls_to_process> 
     198        <database_retry_delay description="If database support is enabled, this value indicates how often (in seconds) the Manager will retry contacting the database, when it is ready for more URLs to process and no URLs were provided during the first request." default="2"> 
     199            2 
     200        </database_retry_delay> 
     201        <command_line_base_priority description="When the Manager is supplied URLs from the command line, it will assign each URL the following numerical base priority." default="1000"> 
     202            1000 
     203        </command_line_base_priority> 
    197204        <!-- HoneyClient::Manager::Database Options --> 
    198205        <Database> 

  • honeyclient/branches/exp/kindlund-simpler_agent/lib/HoneyClient/Manager.pm

    r1419 r1490  
    4242 
    4343  use HoneyClient::Manager; 
    44   use Data::Dumper; 
    45  
    46   # Utility functions to encode configuration data. 
    47   use Storable qw(nfreeze thaw); 
    48   use MIME::Base64 qw(encode_base64 decode_base64); 
    49  
    50   # Note: Make sure only one of these "my driver =" lines 
     44 
     45  # Note: Make sure only one of these "my $driver_name =" lines 
    5146  # is uncommented. 
    5247 
    5348  # Use Internet Explorer as the instrumenting application. 
    54   my $driver = "HoneyClient::Agent::Driver::Browser::IE"; 
     49  my $driver_name = "HoneyClient::Agent::Driver::Browser::IE"; 
    5550 
    5651  # Use Mozilla Firefox as the instrumenting application. 
    57   #my $driver = "HoneyClient::Agent::Driver::Browser::FF"; 
     52  #my $driver_name = "HoneyClient::Agent::Driver::Browser::FF"; 
    5853 
    5954  # Start the Manager. 
    6055  HoneyClient::Manager->run( 
    6156 
    62       driver => $driver, 
    63  
    64       agent_state => encode_base64(nfreeze({ 
    65  
    66           $driver => { 
    67  
    68               # Specify the next link for the Agent VM to visit. 
    69               next_link_to_visit => "http://www.mitre.org", 
    70  
    71               # If you have more than one link, you can also 
    72               # set this type of hashtable: 
    73               links_to_visit => { 
    74                   'http://www.google.com' => 1, 
    75               }, 
    76           }, 
    77  
    78       })), 
     57      driver_name => $driver_name, 
     58 
     59      # The URLs (and priorities) of each entry to process. 
     60      work => { 
     61          'http://www.google.com' => 1, 
     62      }, 
    7963  ); 
    8064 
    …… 
    8468running, and suspending all Agent VMs.  Upon calling the run() function, 
    8569the Manager will proceed to create a new clone of the master Honeyclient VM 
    86 (aka. an Agent VM) and feed this Agent VM a new list of URLs to crawl. 
    87  
    88 While the Agent VM is crawling, the Manager will check to make sure the 
     70(aka. an Agent VM) and feed this Agent VM a new list of URLs to visit. 
     71 
     72While the Agent VM is running, the Manager will check to make sure the 
    8973Agent VM has not been compromised.  If no compromise was found, then the 
    9074Manager will signal the Firewall to allow the Agent VM to contact the 
    …… 
    9478Manager will suspend the clone VM, log the incident, and create a new Agent 
    9579VM clone -- where this new clone picks up with the next set of URLs to 
    96 crawl. 
     80visit. 
    9781 
    9882If there are no URLs left for the Agent VM to visit OR if the user 
    9983presses CTRL+C while the Manager is running, then the Manager will 
    100 suspend the currently running Agent VM and write its state information 
    101 out to the filesystem on the host system.  This file is usually 
    102 called 'Manager.dump'; however, the name can be changed by editing 
    103 the <HoneyClient/><Manager/><manager_state/> section of the 
    104 etc/honeyclient.xml file. 
    105  
    106 This 'Manager.dump' file contains the set of URLs that the Honeyclients 
    107 have visited, ignored, or tried to visit.  In order to determine 
    108 which URLs were identified as malicious, you will need to check 
    109 the syslog on the host system and search for the keyword of "FAILED". 
     84suspend the currently running Agent VM. 
     85 
     86In order to determine which URLs were identified as malicious, you 
     87will need to check the syslog on the host system and search for the 
     88keyword of "FAILED" or "Failure". 
    11089 
    11190By default, all cloned VMs that the Manager suspends will have been 
    112 flagged as compromised -- unless the set of URLs has been exhausted 
     91flagged as suspicious -- unless the set of URLs has been exhausted 
    11392or the user prematurely terminates the process (by pressing CTRL+C). 
    11493 
    …… 
    137116 
    138117    # Symbols to export automatically 
    139     @EXPORT = qw(init destroy); 
     118    @EXPORT = qw(); 
    140119 
    141120    # Items to export into callers namespace by default. Note: do not export 
    …… 
    148127 
    149128    %EXPORT_TAGS = ( 
    150         'all' => [ qw(init destroy) ], 
     129        'all' => [ qw() ], 
    151130    ); 
    152131 
    …… 
    239218use HoneyClient::Util::Config qw(getVar); 
    240219 
    241 # Check if HoneyClient::Manager::Database support is enabled.  
    242 my $DB_ENABLE = getVar(name      => "enable", 
    243                        namespace => "HoneyClient::Manager::Database"); 
    244 if ($DB_ENABLE) { 
    245     # Make sure HoneyClient::Manager::Database loads. 
    246     require_ok('HoneyClient::Manager::Database'); 
    247     require HoneyClient::Manager::Database; 
    248 } 
     220# Make sure HoneyClient::Manager::Database loads. 
     221BEGIN { use_ok('HoneyClient::Manager::Database') or diag("Can't load HoneyClient::Manager::Database package.  Check to make sure the package library is correctly listed within the path."); } 
     222require_ok('HoneyClient::Manager::Database'); 
     223use HoneyClient::Manager::Database; 
    249224 
    250225# Make sure Storable loads. 
    …… 
    262237use MIME::Base64 qw(encode_base64 decode_base64); 
    263238 
     239# Make sure Data::Dumper loads 
     240BEGIN { use_ok('Data::Dumper') 
     241        or diag("Can't load Data::Dumper package. Check to make sure the package library is correctly listed within the path."); } 
     242require_ok('Data::Dumper'); 
     243use Data::Dumper; 
     244 
     245# Make sure Filesys::DfPortable loads 
     246BEGIN { use_ok('Filesys::DfPortable') 
     247        or diag("Can't load Filesys::DfPortable package. Check to make sure the package library is correctly listed within the path."); } 
     248require_ok('Filesys::DfPortable'); 
     249use Filesys::DfPortable; 
     250 
    264251=end testing 
    265252 
    …… 
    283270use HoneyClient::Manager::VM::Clone; 
    284271 
    285 # XXX: Remove this, eventually. 
    286 # TODO: Include unit tests. 
    287 #use HoneyClient::Manager::VM qw(); 
    288  
    289 # Check if HoneyClient::Manager::Database support is enabled.  
    290 our $DB_ENABLE = getVar(name      => "enable", 
    291                         namespace => "HoneyClient::Manager::Database"); 
    292 our $clientDbId = 0; 
    293  
    294 if ($DB_ENABLE) { 
    295     require HoneyClient::Manager::Database; 
    296 } 
    297  
    298 # XXX: Remove this, eventually. 
     272# Include Database Libraries 
     273use HoneyClient::Manager::Database; 
     274 
     275# Use Dumper Library 
    299276use Data::Dumper; 
    300277 
    …… 
    312289use Storable qw(nfreeze thaw); 
    313290 
    314 # TODO: Include unit tests. 
    315 use IO::File; 
    316  
    317 # TODO: Include unit tests. 
    318 use DateTime::HiRes; 
    319  
    320 # TODO: Include unit tests. 
    321 use Sys::Hostname::Long; 
    322  
    323 # TODO: Include unit tests. 
    324 use Sys::HostIP; 
    325  
    326 # TODO: Include unit tests. 
     291# Include Disk Utilization Library 
    327292use Filesys::DfPortable; 
    328293 
    …… 
    332297# The global logging object. 
    333298our $LOG = get_logger(); 
    334  
    335 # This is a temporary, shared variable, used to print out the 
    336 # state of the agent, when _cleanup() occurs. 
    337 # XXX: This variable and all reference to it will be deleted, 
    338 # eventually. 
    339 our $globalAgentState   = undef; 
    340  
    341 #This variable is used to count how many times stubAgent's fault 
    342 #handler has been called, so that special actions can be taken if 
    343 #it is called too many times (for instance when Manager loses  
    344 #connectivity with the Agent it would otherwise loop and get errors 
    345 #forever if some action isn't taken) 
    346 #NOTE: This will have to be changed to be agent/vm-specific in the 
    347 #future when we have multiple Agents interacting with a single  
    348 #Manager. 
    349 our $globalAgentErrorCount = 0; 
    350  
    351 # Temporary variable, used to indicate to the fault handler whether 
    352 # or not errors/warnings should be suppressed. 
    353 our $SUPPRESS_ERRORS = 0; 
    354299 
    355300####################################################################### 
    …… 
    435380####################################################################### 
    436381 
    437 sub _agentHandleFault { 
    438  
    439     # Extract arguments. 
    440     my ($class, $res) = @_; 
    441  
    442  
    443     #NOTE: In the future we may want to have this check first to see if 
    444     #the error is specific to a failed connection (by regexing the error 
    445     #message. But for now, we have no evidence that multiple errors will 
    446     #occur in other circumstances. 
    447     $globalAgentErrorCount++; 
    448  
    449     # Construct error message. 
    450     # Figure out if the error occurred in transport or over 
    451     # on the other side. 
    452     my $errMsg = $class->transport->status; # Assume transport error. 
    453  
    454     if (ref $res) { 
    455         $errMsg = $res->faultcode . ": ".  $res->faultstring . "\n"; 
    456     } 
    457  
    458     if (!$SUPPRESS_ERRORS) { 
    459         $LOG->warn("Error occurred during processing. " . $errMsg); 
    460         Carp::carp __PACKAGE__ . "->_handleFault(): Error occurred during processing.\n" . $errMsg; 
    461     } 
    462 } 
    463  
    464 sub _handleFault { 
    465  
    466     # Extract arguments. 
    467     my ($class, $res) = @_; 
    468  
    469     # Construct error message. 
    470     # Figure out if the error occurred in transport or over 
    471     # on the other side. 
    472     my $errMsg = $class->transport->status; # Assume transport error. 
    473  
    474     if (ref $res) { 
    475         $errMsg = $res->faultcode . ": ".  $res->faultstring . "\n"; 
    476     } 
    477  
    478     if (!$SUPPRESS_ERRORS) { 
    479         $LOG->warn("Error occurred during processing. " . $errMsg); 
    480         Carp::carp __PACKAGE__ . "->_handleFault(): Error occurred during processing.\n" . $errMsg; 
    481     } 
    482 } 
    483  
    484 sub _handleFaultAndCleanup { 
    485  
    486     # Extract arguments. 
    487     my ($class, $res) = @_; 
    488  
    489     # Print fault. 
    490     _handleFault($class, $res); 
    491      
    492     # Cleanup before dying. 
    493     _cleanup(); 
    494 } 
    495  
    496 sub _cleanup { 
    497  
    498     $LOG->info("Cleaning up."); 
    499  
    500     # Mask all possible signals, so that we don't call this function multiple times. 
    501     $SIG{HUP}     = sub { }; 
    502     $SIG{INT}     = sub { }; 
    503     $SIG{QUIT}    = sub { }; 
    504     $SIG{ABRT}    = sub { }; 
    505     $SIG{PIPE}    = sub { }; 
    506     $SIG{TERM}    = sub { }; 
    507  
    508     # XXX: Need to clean this up. 
    509     my $stubFW = getClientHandle(namespace     => "HoneyClient::Manager::FW", 
    510                                  fault_handler => \&_handleFault); 
    511  
    512     # XXX: Change this to installDefaultRules(), eventually. 
    513     # Reset the firewall, to allow everything open. 
    514     $stubFW->allowAllTraffic(); 
    515  
    516     # This variable may contain a filename that the Manager 
    517     # would use to dump its entire state information, upon termination. 
    518     # XXX: May want to change this format/usage, eventually. 
    519     my $STATE_FILE = getVar(name => "manager_state"); 
    520  
    521     if (length($STATE_FILE) > 0 && 
    522         defined($globalAgentState)) { 
    523         $LOG->info("Saving state to '" . $STATE_FILE . "'."); 
    524         my $dump_file = new IO::File($STATE_FILE, "a"); 
    525  
    526         # XXX: Delete this block, eventually. 
    527         $Data::Dumper::Terse = 0; 
    528         $Data::Dumper::Indent = 2; 
    529         print $dump_file Dumper(thaw(decode_base64($globalAgentState))); 
    530         $dump_file->close(); 
    531     } 
    532  
    533     if ($DB_ENABLE && ($clientDbId > 0)) { 
    534         if (defined($globalAgentState)) { 
    535             $LOG->info("Saving URL History to Database."); 
    536             insert_url_history(agent_state => $globalAgentState, 
    537                                client_id   => $clientDbId); 
    538         } 
    539  
    540         # Mark the VM as suspended within the database. 
    541         HoneyClient::Manager::Database::set_client_suspended($clientDbId); 
    542     } 
    543  
    544  
    545     # XXX: There is an issue where if we try to quit but are in the 
    546     # process of asynchronously archiving a VM, then the async archive 
    547     # process will fail. 
    548  
    549     exit; 
    550 } 
    551  
    552 END { 
    553     # Make sure all processes in our process group our dead. 
    554     kill("KILL", -$$); 
    555 } 
    556  
    557 # XXX: Install the cleanup handler, in case the parent process dies 
    558 # unexpectedly. 
    559 $SIG{HUP}  = \&_cleanup; 
    560 $SIG{INT}  = \&_cleanup; 
    561 $SIG{QUIT} = \&_cleanup; 
    562 $SIG{ABRT} = \&_cleanup; 
    563 $SIG{PIPE} = \&_cleanup; 
    564 $SIG{TERM} = \&_cleanup; 
    565  
    566 ####################################################################### 
    567 # Public Methods Implemented                                          # 
    568 ####################################################################### 
    569  
    570 =pod 
    571  
    572 =head1 EXPORTS 
    573  
    574 =head2 run() 
    575  
    576 =over 4 
    577  
    578 # XXX: Fill this in. 
    579  
    580 I<Inputs>:  
    581  B<$arg> is an optional argument. 
    582  
    583 driver 
    584 master_vm_config 
    585 start_state 
    586   
    587 I<Output>: XXX: Fill this in. 
    588  
    589 =back 
    590  
    591 #=begin testing 
    592 # 
    593 # XXX: Fill this in. 
    594 # 
    595 #=end testing 
    596  
    597 =cut 
    598  
    599 sub run { 
    600     # Extract arguments. 
    601     # Hash-based arguments are used, since HoneyClient::Util::SOAP is unable to handle 
    602     # hash references directly.  Thus, flat hashtables are used throughout the code 
    603     # for consistency. 
    604     my ($class, %args) = @_; 
    605     my $agentState = undef; 
    606  
    607     # Sanity check, make sure the master_vm_config is 
    608     # set. 
    609     my $argsExist = scalar(%args); 
    610     if (!$argsExist || 
    611         !exists($args{'master_vm_config'}) || 
    612         !defined($args{'master_vm_config'})) { 
    613         # Get the master_vm_config from the configuration file. 
    614         $args{'master_vm_config'} = getVar(name      => "master_vm_config", 
    615                                            namespace => "HoneyClient::Manager::VM"); 
    616     } 
    617  
    618     for (;;) { 
    619         print "Starting new session...\n"; 
    620         $agentState = $class->runSession(%args); 
    621         $args{'agent_state'} = $agentState; 
    622  
    623         # XXX: Fix this, eventually. 
    624         $globalAgentState = $agentState; 
    625     } 
    626 } 
    627  
    628 sub runSession { 
    629  
    630     # Extract arguments. 
    631     # Hash-based arguments are used, since HoneyClient::Util::SOAP is unable to handle 
    632     # hash references directly.  Thus, flat hashtables are used throughout the code 
    633     # for consistency. 
    634     my ($class, %args) = @_; 
    635  
    636     # XXX: Remove some of these, eventually. 
    637     my $stubFW    = undef; 
    638     my $stubAgent = undef; 
    639     my $som       = undef; 
    640     my $ret       = undef; 
    641     # XXX: Need to figure out a way to move this data into the VM object. 
    642     my $vmCompromised = 0; 
    643     my $vmStateTable = { }; 
    644  
    645     # Temporary variable to hold each cloned VM. 
    646     my $vm        = undef; 
    647  
    648     # Get a stub connection to the firewall. 
    649     $stubFW = getClientHandle(namespace     => "HoneyClient::Manager::FW", 
    650                               fault_handler => \&_handleFaultAndCleanup); 
    651  
    652     # Open up the firewall initially, to allow the Agent to do an SVN update. 
    653     #FIXME: This needs to be more limited for the multi-vm case, and should probably  
    654     # just be included by making the default rules require no action 
    655     $stubFW->allowAllTraffic(); 
    656  
    657     # Check disk space. 
    658     checkSpaceAvailable(); 
    659  
    660     # Create a new cloned VM. 
    661     $vm = HoneyClient::Manager::VM::Clone->new(); 
    662  
    663     #Register Client with the Honeyclient Database 
    664     if ($DB_ENABLE) { 
    665         eval { 
    666             dbRegisterClient($vm); 
    667             $clientDbId = $vm->database_id; 
    668         }; 
    669         if ($@ || ($vm->database_id == 0) || !defined($vm->database_id)) { 
    670             $vm->database_id(0); #$DB_FAILURE 
    671             $LOG->warn("Failure Inserting Client Object:\n$@"); 
    672         } 
    673     } 
    674  
    675     # Build our VM's connection table. 
    676     # Note: We assume our VM has a single MAC address 
    677     # and a single IP address. 
    678     $vmStateTable->{$vm->name}->{sources}->{$vm->mac_address}->{$vm->ip_address} = { 
    679         # XXX: We assume we can't pinpoint what source TCP ports the 
    680         # corresponding driver will need.  (We may want to get this 
    681         # information eventually from the Agent, as part of Driver::next().) 
    682         'tcp' => [80,443], 
    683     }; 
    684  
    685     print "VM State Table:\n"; 
    686     # Make Dumper format more verbose. 
    687     $Data::Dumper::Terse = 0; 
    688     $Data::Dumper::Indent = 2; 
    689     print Dumper($vmStateTable) . "\n"; 
    690    
    691     # Initialize the firewall. 
    692     $stubFW->installDefaultRules(); 
    693  
    694     # Add new chain, per cloned VM. 
    695     $stubFW->addChain($vmStateTable); 
    696     
    697     sleep (2); 
    698  
    699     # Recreate the client stub; handle faults. 
    700     $stubAgent = getClientHandle(namespace     => "HoneyClient::Agent", 
    701                                  address       => $vm->ip_address, 
    702                                  fault_handler => \&_handleFaultAndCleanup); 
    703  
    704     # If supported, get a URL list from the database. 
    705     if ($DB_ENABLE && ($vm->database_id > 0)) { 
    706         $args{'agent_state'} = get_urls($vm, $args{'agent_state'}, $args{'driver'}); 
    707     } 
    708  
    709     # Call updateState() first, to seed initial data. 
    710     # TODO: Need to support asynchronous updates (url adding) 
    711     # from user input. 
    712     print "Calling updateState()...\n"; 
    713     $som = $stubAgent->updateState($args{'agent_state'}); 
    714  
    715     # Recreate the client stub; ignore faults. 
    716     $stubAgent = getClientHandle(namespace     => "HoneyClient::Agent", 
    717                                  address       => $vm->ip_address, 
    718                                  fault_handler => \&_agentHandleFault); 
    719  
    720     # Recreate the firewall stub; ignore faults. 
    721     $stubFW = getClientHandle(namespace     => "HoneyClient::Manager::FW", 
    722                               fault_handler => \&_handleFault); 
    723  
    724     for (my $counter = 1;; $counter++) { 
    725  
    726         # XXX: This isn't a valid assumption! 
    727         # From this point on, catch all errors generated and 
    728         # assume that the Agent's watchdog process will recover. 
    729         eval { 
    730             print "Calling getStatus()...\n"; 
    731             $som = $stubAgent->getStatus(); 
    732             print "Result:\n"; 
    733             $ret = thaw(decode_base64($som->result())); 
    734             # Make Dumper format more verbose. 
    735             $Data::Dumper::Terse = 0; 
    736             $Data::Dumper::Indent = 2; 
    737             print Dumper($ret->{$args{'driver'}}->{status}); 
    738             #print Dumper($ret); 
    739  
    740             # Derive current agent state from full status. 
    741             my @driverNames = keys(%{$ret}); 
    742             my $state = {}; 
    743             foreach my $driverName (@driverNames) { 
    744                 $state->{$driverName} = $ret->{$driverName}->{'state'}; 
    745             } 
    746             $args{'agent_state'} = encode_base64(nfreeze($state)); 
    747             $globalAgentState = $args{'agent_state'}; 
    748             #print "GlobalAgentState:\n"; 
    749             #print Dumper(thaw(decode_base64($globalAgentState))); 
    750  
    751             # Check to see if Agent::run() thread has stopped 
    752             # and that a compromise was detected. 
    753             if (!$ret->{$args{'driver'}}->{status}->{is_running}) { 
    754                 if ($ret->{$args{'driver'}}->{status}->{is_compromised}) { 
    755                     # Check to see if the VM has been compromised. 
    756                     print "WARNING: VM HAS BEEN COMPROMISED!\n"; 
    757                     my $vmName = $vm->name; 
    758                     $vmCompromised = 1; 
    759  
    760                     my $fingerprint = $ret->{$args{'driver'}}->{status}->{fingerprint}; 
    761                     $LOG->warn("VM Compromised. Last Resource (" . $fingerprint->{'last_resource'} . ")"); 
    762  
    763                     # Dump the fingerprint to a file, if needed. 
    764                     # XXX: May want to change this format/usage, eventually. 
    765                     my $COMPROMISE_FILE = getVar(name => "fingerprint_dump"); 
    766                     if (length($COMPROMISE_FILE) > 0 && 
    767                         defined($fingerprint)) { 
    768                         $LOG->info("Saving fingerprint to '" . $COMPROMISE_FILE . "'."); 
    769                         my $dump_file = new IO::File($COMPROMISE_FILE, "a"); 
    770  
    771                         # XXX: Delete this block, eventually. 
    772                         $Data::Dumper::Terse = 0; 
    773                         $Data::Dumper::Indent = 2; 
    774                         print $dump_file "\$vmName = \"" . $vmName . "\";\n"; 
    775                         print $dump_file Dumper($fingerprint); 
    776                         $dump_file->close(); 
    777                     } 
    778  
    779                     # Archive the VM. 
    780                     $LOG->info("Archiving VM..."); 
    781                     $vm->archive(); 
    782  
    783                     # Insert Compromised Fingerprint into DB. 
    784                     if ($DB_ENABLE && ($vm->database_id > 0)) { 
    785                         # Put URL History in database. 
    786                         $LOG->info("Saving URL History to Database."); 
    787                         $args{'agent_state'} = insert_url_history(agent_state => $args{'agent_state'}, 
    788                                                                   client_id   => $vm->database_id); 
    789                         $globalAgentState = $args{'agent_state'}; 
    790                     
    791                         # Delete the 'last_resource' attribute. 
    792                         delete $fingerprint->{last_resource}; 
    793  
    794                         # Associate the client who has this fingerprint. 
    795                         $fingerprint->{client_id} = $vm->database_id; 
    796  
    797                         $LOG->info("Inserting Fingerprint Into Database."); 
    798                         my $fingerprint_id = undef; 
    799                         eval { 
    800                             $fingerprint_id = HoneyClient::Manager::Database::insert_fingerprint($fingerprint); 
    801                         }; 
    802                         if ($@ || ($fingerprint_id == 0) || !defined($fingerprint_id)) { 
    803                             $LOG->warn("Failure Inserting Fingerprint Object:\n$@"); 
    804                         } 
    805  
    806                         $LOG->info("Database Insert Successful."); 
    807                     } 
    808                     # The VM should be suspended, at this point.  Clear out the global DB ID, so 
    809                     # that our cleanup code doesn't re-mark the VM as suspended. 
    810                     $clientDbId = 0; 
    811  
    812                     # Make sure VM is suspended. 
    813                     $vm = undef; 
    814  
    815                     return; # Return out of eval block. 
    816                 } else { 
    817                     print "VM Integrity Check: OK!\n"; 
    818  
    819                     # Check to see if any links remain to be processed by the 
    820                     # Agent. 
    821                     if (!$ret->{$args{'driver'}}->{status}->{links_remaining}) { 
    822      
    823                         # If supported, get more URLs from the database. 
    824                         if ($DB_ENABLE && ($vm->database_id > 0)) { 
    825                             # Put URL History in database. 
    826                             $LOG->info("Saving URL History to Database."); 
    827                             $args{'agent_state'} = insert_url_history(agent_state => $args{'agent_state'}, 
    828                                                                       client_id   => $vm->database_id); 
    829  
    830                             $args{'agent_state'} = get_urls($vm, $args{'agent_state'}, $args{'driver'}); 
    831                             $globalAgentState = $args{'agent_state'}; 
    832                             print "Calling updateState()...\n"; 
    833                             $som = $stubAgent->updateState($args{'agent_state'}); 
    834                         } else { 
    835                             $LOG->info("All URLs exhausted. Shutting down Manager."); 
    836                             $vm = undef; 
    837                             _cleanup(); 
    838                         } 
    839                     } else { 
    840                         # The Agent::run() thread has stopped; we assume 
    841                         # it's because the Agent is waiting for the firewall 
    842                         # to allow access to the new targets. 
    843                  
    844                         # Delete the old firewall rules, based upon existing 
    845                         # targets. 
    846                         $stubFW->deleteRules($vmStateTable); 
    847  
    848                         # Get the new targets from the Agent. 
    849                         $vmStateTable->{$vm->name}->{targets} = $ret->{$args{'driver'}}->{next}->{targets}; 
    850                         #$vmStateTable->{$vm->name}->{targets} = '0.0.0.0'; 
    851  
    852                         print "VM State Table:\n"; 
    853                         # Make Dumper format more verbose. 
    854                         $Data::Dumper::Terse = 0; 
    855                         $Data::Dumper::Indent = 2; 
    856                         print Dumper($vmStateTable) . "\n"; 
    857  
    858                         # Add the new targets from the Agent. 
    859                         $stubFW->addRules($vmStateTable); 
    860  
    861                         print "Calling run()...\n"; 
    862                         $som = $stubAgent->run(driver_name => $args{'driver'}); 
    863                     } 
    864                 } 
    865             } 
    866         }; 
    867         if ($@) { 
    868             print "Error: $@\n"; 
    869             my $resetSuccessful = 0; 
    870             while (!$resetSuccessful) { 
    871                 print "Resetting firewall...\n"; 
    872                 eval { 
    873                     # We assume the error was caused by some sort of communications 
    874                     # problem with the Agent.  Assume the Agent's watchdog will restart 
    875                     # the daemon, in which case, we indefinately try to reset the 
    876                     # firewall accordingly. 
    877                     $stubFW->installDefaultRules(); 
    878                     $stubFW->addChain($vmStateTable); 
    879                     $stubFW->addRules($vmStateTable); 
    880                 }; 
    881                 if (!$@) { 
    882                     $resetSuccessful = 1; 
    883                 } else { 
    884                     sleep (3); 
    885                 } 
    886             }    
    887         } 
    888         if ($vmCompromised) { 
    889             # Reset the FW state table.  
    890             $vmStateTable = ( ); 
    891             return $args{'agent_state'}; 
    892         } 
    893         if ($globalAgentErrorCount >= getVar(name => "max_agent_error_count")) { 
    894             if ($DB_ENABLE && ($vm->database_id > 0)) { 
    895                 # Mark the VM as suspended within the database. 
    896                 my $dt = DateTime::HiRes->now(time_zone => "local"); 
    897                 HoneyClient::Manager::Database::set_client_suspicious({ 
    898                     client_id => $vm->database_id, 
    899                     compromise => $dt->ymd('-').'T'.$dt->hms(':'), 
    900                 }); 
    901             } 
    902             # The VM should be suspended after the return.  Clear out the global DB ID, so 
    903             # that our cleanup code doesn't re-mark the VM as suspended. 
    904             $clientDbId = 0; 
    905  
    906             $globalAgentErrorCount = 0; 
    907             # Reset the FW state table.  
    908             $vmStateTable = ( ); 
    909             return $args{'agent_state'}; 
    910         } 
    911         print "Sleeping for 2s...\n"; 
    912         sleep (2); 
    913     } 
    914 } 
    915  
    916 sub insert_url_history { 
    917  
    918     # Extract arguments. 
    919     my %args = @_; 
    920      
    921     my $state = thaw(decode_base64($args{'agent_state'})); 
    922     my $driver = undef; 
    923     foreach my $key (keys %$state) { 
    924         if ($state->{$key}) { 
    925             $driver = $key;  
    926             last; 
    927         } 
    928     } 
    929  
    930     # Set the client ID. 
    931     $state->{$driver}->{'client_id'} = $args{'client_id'}; 
    932     
    933     # XXX: Delete this, eventually. 
    934     #use Data::Dumper; 
    935     #$LOG->info("agent_state = " . Data::Dumper::Dumper($state)); 
    936  
    937     my $num_urls_inserted = HoneyClient::Manager::Database::insert_history_urls($state->{$driver}); 
    938     $LOG->info($num_urls_inserted . " URL(s) Inserted."); 
    939  
    940     # Flush the URL history, after committing to the database. 
    941     $state->{$driver}->{'links_visited'} = {}; 
    942     return encode_base64(nfreeze($state)); 
    943 } 
    944  
    945 sub dbRegisterClient { 
    946     my $vm = shift; 
    947     my $dt = DateTime::HiRes->now(time_zone => "local"); 
    948  
    949     # Register the VM with the DB 
    950     my $client = { 
    951         cid => $vm->name, 
    952         status => 'running', 
    953         host => { 
    954             org => getVar(name => "organization"), 
    955             hostname => Sys::Hostname::Long::hostname_long, 
    956             ip => Sys::HostIP->ip, 
    957         }, 
    958         os => { 
    959             name => 'Default Windows XP SP2', 
    960             shortname => 'Microsoft Windows', 
    961             version => 'XP Professional', 
    962             #os_patches => [{ 
    963             #    name => 'Service Pack 2', 
    964             #}], 
    965             os_applications => [{ 
    966                 manufacturer => 'Microsoft', 
    967                 shortname => 'Internet Explorer', 
    968                 version => '6', 
    969             }], 
    970         }, 
    971         start => $dt->ymd('-').'T'.$dt->hms(':'), 
    972     }; 
    973     $vm->database_id(HoneyClient::Manager::Database::insert_client($client)); 
    974 } 
    975  
    976 sub get_urls { 
    977     my $vm = shift; 
    978     my $agent_state = shift; 
    979     my $driver = shift; 
    980  
    981     # Decode and thaw the initial agent state. 
    982     my $state = thaw(decode_base64($agent_state)); 
    983  
    984     my $queue_url_list = {}; 
    985     $LOG->info("Waiting for new URLs from database."); 
    986     # XXX: We hardcode the value of 10 URLs to request; this will change, eventually. 
    987     $queue_url_list = HoneyClient::Manager::Database::get_queue_urls(10, $vm->database_id); 
    988     my $remoteLinksExist = scalar(%{$queue_url_list}); 
    989  
    990     # While we have no local URLs and no URLs from the database, re-query the database. 
    991     while (!defined($state->{$driver}->{next_link_to_visit}) && 
    992            !$remoteLinksExist) { 
    993  
    994         # XXX: Hardcoded timeout. 
    995         sleep (2); 
    996         # XXX: Trap/ignore all errors and simply retry. 
    997         eval { 
    998             $queue_url_list = HoneyClient::Manager::Database::get_queue_urls(10, $vm->database_id); 
    999             $remoteLinksExist = scalar(%{$queue_url_list}); 
    1000         }; 
    1001     } 
    1002  
    1003     # If we do have URLs from t