Changeset 1087

Timestamp:

12/30/07 14:50:17 (11 months ago)

Author:

xkovah

Message:

still underway…just checking in to save

Files:

• honeyclient/branches/exp/xkovah-priority_interrupt/etc/honeyclient.xml (modified) (1 diff)
• honeyclient/branches/exp/xkovah-priority_interrupt/lib/HoneyClient/Agent.pm (modified) (1 diff)
• honeyclient/branches/exp/xkovah-priority_interrupt/lib/HoneyClient/Agent/Driver/Browser.pm (modified) (4 diffs)
• honeyclient/branches/exp/xkovah-priority_interrupt/lib/HoneyClient/Manager-local.pm (added)
• honeyclient/branches/exp/xkovah-priority_interrupt/lib/HoneyClient/Manager-orig.pm (added)
• honeyclient/branches/exp/xkovah-priority_interrupt/lib/HoneyClient/Manager.pm (modified) (29 diffs)

honeyclient/branches/exp/xkovah-priority_interrupt/etc/honeyclient.xml

@@ -377 +377 @@
-            3
-        </max_agent_error_count>
+        <priority_links_file description="File path (absolute, or relative to manager) to a file which can contain links which should be visited ASAP, by interrupting the existing agent which is running" default="../../../priority_links.txt">
+        ../../../priority_links.txt
+        </priority_links_file>
-        <!-- HoneyClient::Manager::FW Options -->
-        <FW>

honeyclient/branches/exp/xkovah-priority_interrupt/lib/HoneyClient/Agent.pm

@@ -1189 +1189 @@
-}
-
+
+sub setPriorityLinksStatus{
+    my ($junk, $val) = @_;
+    my $data = _lock();
+
+    
+    _unlock($data);
+}
+
-# XXX: Document this.
-# XXX: Do we really need this?

honeyclient/branches/exp/xkovah-priority_interrupt/lib/HoneyClient/Agent/Driver/Browser.pm

@@ -451 +451 @@
-#
-# When getting the next link, 'next_link_to_visit' is checked first.
-
+
+
-# is checked next.  If that hashtable is empty, then finally the
-# 'links_to_visit' hashtable is checked.
@@ -898 +899 @@
-        links_timed_out         => { },
-
+        #
+        priority_links  => { },
+
+
-        # If this parameter is a defined scalar, then the browser
-        # will also never attempt to revisit any links that caused
@@ -1648 +1653 @@
-    return (!(defined($self->next_link_to_visit) or
-              scalar(%{$self->relative_links_to_visit}) or
+              scalar(%{$self->priority_links}) or
-              scalar(%{$self->links_to_visit})))
-
@@ -1727 +1733 @@
-                                          $next_link_is_set;
-
+    # Figure out how many priority links are left to process.
+    #FIXME: Should this have + $next_link_is_set?
+    $status->{priority_links_remaining} = scalar(keys(%{$self->priority_links})) + $next_link_is_set;
+    
-    # Figure out how many total links are left to process.
-
-
-
+
+
+
+
+
+
+
+
+
-    # Set the total number of links in the object's state.
-    $status->{links_total} = $status->{links_processed} +

honeyclient/branches/exp/xkovah-priority_interrupt/lib/HoneyClient/Manager.pm

@@ -28 +28 @@
-#######################################################################
-
-=pod
-
-=head1 NAME
-
-HoneyClient::Manager - Perl extension to manage Agent VMs on the
-host system.
-
-=head1 VERSION
-
-This documentation refers to HoneyClient::Manager version 1.00.
-
-=head1 SYNOPSIS
-
-  use HoneyClient::Manager;
-  use Data::Dumper;
-
-  # Utility functions to encode configuration data.
-  use Storable qw(nfreeze thaw);
-  use MIME::Base64 qw(encode_base64 decode_base64);
-
-  # Note: Make sure only one of these "my driver =" lines
-  # is uncommented.
-
-  # Use Internet Explorer as the instrumenting application.
-  my $driver = "HoneyClient::Agent::Driver::Browser::IE";
-
-  # Use Mozilla Firefox as the instrumenting application.
-  #my $driver = "HoneyClient::Agent::Driver::Browser::FF";
-
-  # Start the Manager.
-  HoneyClient::Manager->run(
-
-      driver => $driver,
-
-      agent_state => encode_base64(nfreeze({
-
-          $driver => {
-
-              # Specify the next link for the Agent VM to visit.
-              next_link_to_visit => "http://www.mitre.org",
-
-              # If you have more than one link, you can also
-              # set this type of hashtable:
-              links_to_visit => {
-                  'http://www.google.com' => 1,
-              },
-          },
-
-      })),
-  );
-
-=head1 DESCRIPTION
-
-This module provides centralized control over provisioning, initializing,
-running, and suspending all Agent VMs.  Upon calling the run() function,
-the Manager will proceed to create a new clone of the master Honeyclient VM
-(aka. an Agent VM) and feed this Agent VM a new list of URLs to crawl.
-
-While the Agent VM is crawling, the Manager will check to make sure the
-Agent VM has not been compromised.  If no compromise was found, then the
-Manager will signal the Firewall to allow the Agent VM to contact the
-next set of network resources (i.e., a webserver).
-
-If the Manager discovers the Agent VM has been compromised, then the
-Manager will suspend the clone VM, log the incident, and create a new Agent
-VM clone -- where this new clone picks up with the next set of URLs to
-crawl.
-
-If there are no URLs left for the Agent VM to visit OR if the user
-presses CTRL+C while the Manager is running, then the Manager will
-suspend the currently running Agent VM and write its state information
-out to the filesystem on the host system.  This file is usually
-called 'Manager.dump'; however, the name can be changed by editing
-the <HoneyClient/><Manager/><manager_state/> section of the
-etc/honeyclient.xml file.
-
-This 'Manager.dump' file contains the set of URLs that the Honeyclients
-have visited, ignored, or tried to visit.  In order to determine
-which URLs were identified as malicious, you will need to check
-the syslog on the host system and search for the keyword of "FAILED".
-
-By default, all cloned VMs that the Manager suspends will have been
-flagged as compromised -- unless the set of URLs has been exhausted
-or the user prematurely terminates the process (by pressing CTRL+C).
-
-=cut
-
-package HoneyClient::Manager;
@@ -161 +75 @@
-our (@EXPORT_OK, $VERSION);
-
-=pod
-
-=begin testing
-
-# Make sure ExtUtils::MakeMaker loads.
-BEGIN { use_ok('ExtUtils::MakeMaker', qw(prompt)) or diag("Can't load ExtUtils::MakeMaker package.  Check to make sure the package library is correctly listed within the path."); }
-require_ok('ExtUtils::MakeMaker');
-can_ok('ExtUtils::MakeMaker', 'prompt');
-use ExtUtils::MakeMaker qw(prompt);
-
-# Make sure Log::Log4perl loads
-BEGIN { use_ok('Log::Log4perl', qw(:nowarn))
-        or diag("Can't load Log::Log4perl package. Check to make sure the package library is correctly listed within the path.");
-       
-        # Suppress all logging messages, since we need clean output for unit testing.
-        Log::Log4perl->init({
-            "log4perl.rootLogger"                               => "DEBUG, Buffer",
-            "log4perl.appender.Buffer"                          => "Log::Log4perl::Appender::TestBuffer",
-            "log4perl.appender.Buffer.min_level"                => "fatal",
-            "log4perl.appender.Buffer.layout"                   => "Log::Log4perl::Layout::PatternLayout",
-            "log4perl.appender.Buffer.layout.ConversionPattern" => "%d{yyyy-MM-dd HH:mm:ss} %5p [%M] (%F:%L) - %m%n",
-        });
-}
-require_ok('Log::Log4perl');
-use Log::Log4perl qw(:easy);
-
-# Make sure HoneyClient::Util::Config loads.
-BEGIN { use_ok('HoneyClient::Util::Config', qw(getVar))
-        or diag("Can't load HoneyClient::Util::Config package.  Check to make sure the package library is correctly listed within the path.");
-
-        # Suppress all logging messages, since we need clean output for unit testing.
-        Log::Log4perl->init({
-            "log4perl.rootLogger"                               => "DEBUG, Buffer",
-            "log4perl.appender.Buffer"                          => "Log::Log4perl::Appender::TestBuffer",
-            "log4perl.appender.Buffer.min_level"                => "fatal",
-            "log4perl.appender.Buffer.layout"                   => "Log::Log4perl::Layout::PatternLayout",
-            "log4perl.appender.Buffer.layout.ConversionPattern" => "%d{yyyy-MM-dd HH:mm:ss} %5p [%M] (%F:%L) - %m%n",
-        });
-}
-require_ok('HoneyClient::Util::Config');
-can_ok('HoneyClient::Util::Config', 'getVar');
-use HoneyClient::Util::Config qw(getVar);
-
-# Suppress all logging messages, since we need clean output for unit testing.
-Log::Log4perl->init({
-    "log4perl.rootLogger"                               => "DEBUG, Buffer",
-    "log4perl.appender.Buffer"                          => "Log::Log4perl::Appender::TestBuffer",
-    "log4perl.appender.Buffer.min_level"                => "fatal",
-    "log4perl.appender.Buffer.layout"                   => "Log::Log4perl::Layout::PatternLayout",
-    "log4perl.appender.Buffer.layout.ConversionPattern" => "%d{yyyy-MM-dd HH:mm:ss} %5p [%M] (%F:%L) - %m%n",
-});
-
-# Make sure the module loads properly, with the exportable
-# functions shared.
-BEGIN { use_ok('HoneyClient::Manager', qw(init destroy)) or diag("Can't load HoneyClient::Manager package.  Check to make sure the package library is correctly listed within the path."); }
-require_ok('HoneyClient::Manager');
-can_ok('HoneyClient::Manager', 'init');
-can_ok('HoneyClient::Manager', 'destroy');
-use HoneyClient::Manager qw(init destroy);
-
-# Make sure HonyClient::Manager::VM::Clone loads.
-BEGIN { use_ok('HoneyClient::Manager::VM::Clone') or diag("Can't load HoneyClient::Manager::VM::Clone package.  Check to make sure the package library is correctly listed within the path."); }
-require_ok('HoneyClient::Manager::VM::Clone');
-use HoneyClient::Manager::VM::Clone;
-
-# Make sure HoneyClient::Util::SOAP loads.
-BEGIN { use_ok('HoneyClient::Util::SOAP', qw(getServerHandle getClientHandle)) or diag("Can't load HoneyClient::Util::SOAP package.  Check to make sure the package library is correctly listed within the path."); }
-require_ok('HoneyClient::Util::SOAP');
-can_ok('HoneyClient::Util::SOAP', 'getServerHandle');
-can_ok('HoneyClient::Util::SOAP', 'getClientHandle');
-use HoneyClient::Util::SOAP qw(getServerHandle getClientHandle);
-
-# Make sure HoneyClient::Util::Config loads.
-BEGIN { use_ok('HoneyClient::Util::Config', qw(getVar)) or diag("Can't load HoneyClient::Util::Config package.  Check to make sure the package library is correctly listed within the path."); }
-require_ok('HoneyClient::Util::Config');
-can_ok('HoneyClient::Util::Config', 'getVar');
-use HoneyClient::Util::Config qw(getVar);
-
-# Check if HoneyClient::DB support is enabled. 
-my $DB_ENABLE = getVar(name      => "enable",
-                       namespace => "HoneyClient::DB");
-
-if ($DB_ENABLE) {
-    # Make sure HoneyClient::DB::Fingerprint loads. 
-    require_ok('HoneyClient::DB::Fingerprint');
-    require HoneyClient::DB::Fingerprint;
-}
-
-# Make sure Storable loads.
-BEGIN { use_ok('Storable', qw(nfreeze thaw)) or diag("Can't load Storable package.  Check to make sure the package library is correctly listed within the path."); }
-require_ok('Storable');
-can_ok('Storable', 'nfreeze');
-can_ok('Storable', 'thaw');
-use Storable qw(nfreeze thaw);
-
-# Make sure MIME::Base64 loads.
-BEGIN { use_ok('MIME::Base64', qw(encode_base64 decode_base64)) or diag("Can't load MIME::Base64 package.  Check to make sure the package library is correctly listed within the path."); }
-require_ok('MIME::Base64');
-can_ok('MIME::Base64', 'encode_base64');
-can_ok('MIME::Base64', 'decode_base64');
-use MIME::Base64 qw(encode_base64 decode_base64);
-
-=end testing
-
-=cut
-
-#######################################################################
@@ -281 +90 @@
-use HoneyClient::Util::Config qw(getVar);
-
-# Include the VM Utility Libraries
-use HoneyClient::Manager::VM::Clone;
-
-# XXX: Remove this, eventually.
-# TODO: Include unit tests.
-use HoneyClient::Manager::VM qw();
-
-# Check if HoneyClient::DB support is enabled. 
-our $DB_ENABLE = getVar(name      => "enable",
-                        namespace => "HoneyClient::DB");
-our $clientDbId = 0;
-our %link_categories;
-
-if ($DB_ENABLE) {
-    # Include HoneyClient::DB Utility Libraries
-    # TODO: Include unit tests.
-    require HoneyClient::DB::Fingerprint;
-    require HoneyClient::DB::Client;
-    require HoneyClient::DB::Url::History;
-    require HoneyClient::DB::Time;
-    %link_categories = (
-        $HoneyClient::DB::Url::History::STATUS_VISITED => 'links_visited',
-        $HoneyClient::DB::Url::History::STATUS_TIMED_OUT => 'links_timed_out',
-# For the time being, ignored links will not be inserted.
-#       $HoneyClient::DB::Url::History::STATUS_IGNORED => 'links_ignored',
-    );
-}
-
-# XXX: Remove this, eventually.
@@ -325 +109 @@
-use Storable qw(nfreeze thaw);
-
-# Include VmPerl Constants.
-# TODO: Include unit tests.
-use VMware::VmPerl qw(VM_EXECUTION_STATE_ON
-                      VM_EXECUTION_STATE_OFF
-                      VM_EXECUTION_STATE_STUCK
-                      VM_EXECUTION_STATE_SUSPENDED);
-
-# TODO: Include unit tests.
-use IO::File;
@@ -339 +116 @@
-
-# TODO: Include unit tests.
-
+#
-
-# TODO: Include unit tests.
-
+#
-
-# Include Logging Library
-
-
-
-
+
-
-# Complete URL of SOAP server, when initialized.
@@ -357 +131 @@
-our $DAEMON_PID     : shared = undef;
-
-# XXX: These will be migrated somewhere else, eventually.
-our $vmCloneConfig      = undef;
-
-# This is a temporary, shared variable, used to print out the
@@ -365 +137 @@
-# eventually.
-our $globalAgentState   = undef;
-
-
-handler has been called, so that special actions can be taken if
-it is called too many times (for instance when Manager loses 
-connectivity with the Agent it would otherwise loop and get errors
-forever if some action isn't taken)
-
-
-Manager.
-globalAgentErrorCount = 0
+
+
+our $priority_links_to_visit = 0;
+our %priority_links = ();
+FIXME: This should by putting the links into the agent driver state
+Global error count for determining when to stop trying to contact the agent
+
+
+The file which is checked to see if there are priority links to visit
+priority_links_file = getVar(name => "priority_links_file")
-
-# Temporary variable, used to indicate to the fault handler whether
@@ -468 +240 @@
-
-
-
-
-
-
-
-
+
+
-    # Construct error message.
-    # Figure out if the error occurred in transport or over
@@ -484 +252 @@
-
-    if (!$SUPPRESS_ERRORS) {
-        $LOG->warn("Error occurred during processing. " . $errMsg);
-        Carp::carp __PACKAGE__ . "->_handleFault(): Error occurred during processing.\n" . $errMsg;
-    }
@@ -504 +271 @@
-
-    if (!$SUPPRESS_ERRORS) {
-        $LOG->warn("Error occurred during processing. " . $errMsg);
-        Carp::carp __PACKAGE__ . "->_handleFault(): Error occurred during processing.\n" . $errMsg;
-    }
@@ -523 +289 @@
-sub _cleanup {
-
-    $LOG->info("Cleaning up.");
-
-    # Mask all possible signals, so that we don't call this function multiple times.
@@ -533 +298 @@
-    $SIG{TERM}    = sub { };
-
-    # XXX: Need to clean this up.
-    my $stubFW = getClientHandle(namespace     => "HoneyClient::Manager::FW");
-
-    # XXX: Change this to installDefaultRules(), eventually.
-    # Reset the firewall, to allow everything open.
-    $stubFW->allowAllTraffic();
-
-    # This variable may contain a filename that the Manager
@@ -547 +306 @@
-    if (length($STATE_FILE) > 0 &&
-        defined($globalAgentState)) {
-        $LOG->info("Saving state to '" . $STATE_FILE . "'.");
-        my $dump_file = new IO::File($STATE_FILE, "a");
-
@@ -556 +314 @@
-        $dump_file->close();
-    }
-    #XXX: Insert Urls. To be moved eventually.
-    if ($DB_ENABLE && ($clientDbId > 0)) {
-        $LOG->info("Saving Url History to Database.");
-        insert_url_history(agent_state => $globalAgentState);
-        HoneyClient::DB::Client->update(
-            '-set' => {
-                status => $HoneyClient::DB::Client::STATUS_CLEAN,
-            },
-            '-where' => {
-                id => $clientDbId,
-            }
-        );
-    }
-
-    # XXX: There is an issue where if we try to quit but are in the
@@ -630 +375 @@
-    # set.
-    my $argsExist = scalar(%args);
-
-
-
-
-
-
-
+
+
+
-
-    for (;;) {
@@ -642 +383 @@
-        $agentState = $class->runSession(%args);
-        $args{'agent_state'} = $agentState;
-
-        # XXX: Delete this, eventually.
-        $globalAgentState = $agentState;
-
-
-
-
+
+
+
+
-    }
-}
+
-
-sub runSession {
@@ -661 +403 @@
-
-# XXX: Remove some of these, eventually.
-    my $stubVM    = undef;
-    my $stubFW    = undef;
-    my $stubAgent = undef;
-    my $som       = undef;
-    my $ret       = undef;
-#DELETE ME UNUSED 
-#DELETE ME UNUSED 
-#DELETE ME UNUSED 
+
+
+
-    my $vmCompromised = 0;
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-    # Build our VM's connection table.
-    # Note: We assume our VM has a single MAC address
-    # and a single IP address.
-->{$vm->name}->{sources}->{$vm->mac_address}->{$vm->ip_address
+{"127.0.0.1"
-        # XXX: We assume we can't pinpoint what source TCP ports the
-        # corresponding driver will need.  (We may want to get this
@@ -712 +426 @@
-    $Data::Dumper::Terse = 0;
-    $Data::Dumper::Indent = 2;
-$
+\%
-  
-    # Initialize the firewall.
-    $stubFW->installDefaultRules();
-
-    # Add new chain, per cloned VM.
-    $stubFW->addChain($vmStateTable);
-   
-    sleep (2);
-
-    # Recreate the client stub; handle faults.
-    $stubAgent = getClientHandle(namespace     => "HoneyClient::Agent",
-$vm->ip_address
+"127.0.0.1"
-                                 fault_handler => \&_handleFaultAndCleanup);
-
@@ -731 +439 @@
-    # from user input.
-    print "Calling updateState()...\n";
+    print "YOU MUST BE AT LEAST THIS COMPLETE TO RIDE THE UPDATESTATE\n";
+    print Dumper($args{'agent_state'});
-    $som = $stubAgent->updateState($args{'agent_state'});
-
-    # Recreate the client stub; ignore faults.
-    $stubAgent = getClientHandle(namespace     => "HoneyClient::Agent",
-$vm->ip_address
+"127.0.0.1"
-                                 fault_handler => \&_agentHandleFault);
-
-    # Recreate the firewall stub; ignore faults.
-    $stubFW = getClientHandle(namespace     => "HoneyClient::Manager::FW",
-                              fault_handler => \&_handleFault);
-
-    for (my $counter = 1;; $counter++) {
@@ -748 +454 @@
-        # assume that the Agent's watchdog process will recover.
-        eval {
+
-            print "Calling getState()...\n";
-            $som = $stubAgent->getState();
@@ -761 +468 @@
-            # Make Dumper format more verbose.
-            $Data::Dumper::Terse = 0;
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-            # Check to see if Agent::run() thread has stopped
@@ -771 +496 @@
-                    # Check to see if the VM has been compromised.
-                    print "WARNING: VM HAS BEEN COMPROMISED!\n";
-
-
-
-
+
-                    $vmCompromised = 1;
-
-                    my $fingerprint = $ret->{$args{'driver'}}->{status}->{fingerprint};
-                    $LOG->warn("VM Compromised. Last Resource (" . $fingerprint->{'last_resource'} . ")");
-
-                    # Dump the fingerprint to the compromise file, if needed.
@@ -785 +506 @@
-                    if (length($COMPROMISE_FILE) > 0 &&
-                        defined($fingerprint)) {
-                        $LOG->info("Saving fingerprint to '" . $COMPROMISE_FILE . "'.");
-                        my $dump_file = new IO::File($COMPROMISE_FILE, "a");
-
@@ -796 +516 @@
-                    }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-                    return; # Return out of eval block.
-
+
+
-                    print "VM Integrity Check: OK!\n";
-
@@ -836 +526 @@
-                    if (!$ret->{$args{'driver'}}->{status}->{links_remaining}) {
-
-                        $LOG->info("All URLs exhausted.  Shutting down Manager.");
-                        $vm = undef;
-                        # Get a local copy of the configuration and kill the global copy.
-#                        my $vmCfg = $vmCloneConfig;
-#                        $vmCloneConfig = undef;
-#                        $LOG->info("Calling suspendVM(config => " . $vmCfg . ").");
-#                        $stubVM->suspendVM(config => $vmCfg);
-                        print "Done!\n";
-                        _cleanup();
@@ -851 +535 @@
-                        # to allow access to the new targets.
-                
-                        # Delete the old firewall rules, based upon existing
-                        # targets.
-                        $stubFW->deleteRules($vmStateTable);
-
-                        # Get the new targets from the Agent.
-
-
+
-
-                        print "VM State Table:\n";
@@ -863 +543 @@
-                        $Data::Dumper::Terse = 0;
-                        $Data::Dumper::Indent = 2;
-
-
-
-
+
-
-                        print "Calling run()...\n";
@@ -884 +561 @@
-                    # the daemon, in which case, we indefinately try to reset the
-                    # firewall accordingly.
-                    $stubFW->installDefaultRules();
-                    $stubFW->addChain($vmStateTable);
-                    $stubFW->addRules($vmStateTable);
-                };
-                if (!$@) {
@@ -897 +571 @@
-        if ($vmCompromised) {
-            # Reset the FW state table. 
-$
+%
-            return $args{'agent_state'};
-        }
-globalAgentErrorCount >= getVar(name => "max_agent_error_count")
-globalAgentErrorC
+G_error_count >= 3
+G_error_c
-            # Reset the FW state table. 
-$
+%
-            return $args{'agent_state'};
-        }
+        if(scalar(keys %priority_links) > 0){
+            %vmStateTable = ( );
+            #In this case, we want to suspend the VM and we need to build a 
+            #new agentState which has only the links from the %priority_links
+            $args{agent_state}->links_to_visit = \%priority_links;
+
+ 
+        }
+
-        print "Sleeping for 2s...\n";
-        sleep (2);
+    } #end infinite for loop
+}
+
+#helper for checking for priority links to interrupt the current run session
+sub _updatePriorityLinks {
+
+    $driverRef = shift;
+    $priority_links = $driverRef->{'state'}->{'priority_links'};
+
+    if(-f $priority_links_file && -s $priority_links_file){
+        open(PRIORITY, $priority_links_file);
+        while(<PRIORITY>){
+            my $tmp = $_;
+            chomp($tmp);
+            $tmp =~ s/\r$//;
+            $priority_links{$tmp} = 1; 
+        }
+        close(PRIORITY);
+        system("rm $priority_links_file"); #FIXME: make more generic
-    }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-}
-