Show
Ignore:
Timestamp:
05/14/08 14:58:08 (3 months ago)
Author:
kindlund
Message:

Added work_unit_limit logic; where cloned VMs will get recycled after visiting X number of URLs. This resolves the issue where our driving application may have a slow memory leak; where after X URLs the OS memory is completely filled, rendering the VM unusable. Instead, we preemptively destroy the VM and create a new one before we hit X. Initially, we'll let X = 2000 and go from there.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • honeyclient/trunk/etc/honeyclient.xml

    r1581 r1582  
    350350                    1 
    351351                </archive_upon_suspend> 
     352                <work_unit_limit description="An integer, indicating how many work units (e.g., URLs) the clone VM should process before destroying the VM and regenerating a new clone VM.  This option is useful to set, when trying to drive an application that appears to create slow memory leaks within the VM's OS.  For example, after processing X work units, the VM's OS runs out of memory altogether.  To deal with this issue, we preemptively destroy and regenerate a new clone VM at (X-1) work units, so that the leak no longer affects our operations.  To completely disable this functionality, specify -1." default="2000"> 
     353                    2000 
     354                </work_unit_limit> 
    352355            </Clone> 
    353356            <!-- HoneyClient::Manager::VM::Test Options --> 

  • honeyclient/trunk/lib/HoneyClient/Manager/VM/Clone.pm

    r1554 r1582  
    490490=back 
    491491 
     492=head2 work_units_processed 
     493 
     494=over 4 
     495 
     496The number of work units processed by this VM. 
     497 
     498=back 
     499 
    492500=cut 
    493501 
    …… 
    12771285                              namespace => "HoneyClient::Agent"), 
    12781286 
     1287        # A variable indicating the number of work units processed by this 
     1288        # cloned VM. 
     1289        work_units_processed => 0, 
     1290 
    12791291        # A SOAP handle to the VM manager daemon.  (This internal variable 
    12801292        # should never be modified externally.) 
    …… 
    14431455                       "# Your master VM is: " . getVar(name => "master_vm_config", namespace => "HoneyClient::Manager::VM") . "\n" . 
    14441456                       "#\n" . 
    1445                        "# Do you want to test cloning and archiving this master VM?", "no"); 
     1457                       "# Do you want to test cloning this master VM and archiving a subsequent clone?", "no"); 
    14461458    if ($question =~ /^y.*/i) { 
    14471459 
    …… 
    15431555            $LOG->error("Thread ID (" . threads->tid() . "): Unable to archive VM (" . $vmConfig . ")."); 
    15441556        } 
     1557    } 
     1558 
     1559    return $self; 
     1560} 
     1561 
     1562=pod 
     1563 
     1564=head2 $object->destroy() 
     1565 
     1566=over 4 
     1567 
     1568Destroys an existing Clone object, by powering off the VM and 
     1569deleting the subdirectory containing the VM data. 
     1570 
     1571I<Output>: The destroyed Clone B<$object>. 
     1572 
     1573I<Notes>: 
     1574This operation alters the Clone B<$object>.  Do not 
     1575expect to perform any additional operations with  
     1576this object once this call is finished, since the 
     1577underlying VM has been destroyed. 
     1578 
     1579=back 
     1580 
     1581=begin testing 
     1582 
     1583# Shared test variables. 
     1584my ($stub, $som, $URL); 
     1585my $testVM = $ENV{PWD} . "/" . getVar(name      => "test_vm_config", 
     1586                                      namespace => "HoneyClient::Manager::VM::Test"); 
     1587 
     1588# Catch all errors, in order to make sure child processes are 
     1589# properly killed. 
     1590eval { 
     1591 
     1592    my $testVMDir = dirname($testVM); 
     1593 
     1594    # Pretend as though no other Clone objects have been created prior 
     1595    # to this point. 
     1596    $HoneyClient::Manager::VM::Clone::OBJECT_COUNT = -1; 
     1597     
     1598    my $question; 
     1599    $question = prompt("#\n" . 
     1600                       "# Note: Testing real destroy operations will *ONLY* work\n" . 
     1601                       "# with a fully functional master VM that has the HoneyClient code\n" . 
     1602                       "# loaded upon boot-up.\n" . 
     1603                       "#\n" . 
     1604                       "# This test also requires that the firewall VM is registered,\n" . 
     1605                       "# powered on, and operational.\n" . 
     1606                       "#\n" . 
     1607                       "# Your master VM is: " . getVar(name => "master_vm_config", namespace => "HoneyClient::Manager::VM") . "\n" . 
     1608                       "#\n" . 
     1609                       "# Do you want to test cloning this master VM and destroying a subsequent clone?", "no"); 
     1610    if ($question =~ /^y.*/i) { 
     1611 
     1612        # Create a generic empty clone, with test state data. 
     1613        my $clone = HoneyClient::Manager::VM::Clone->new(_bypass_firewall => 1); 
     1614        my $cloneConfig = $clone->{config}; 
     1615 
     1616        # Archive the clone. 
     1617        $clone->destroy(); 
     1618 
     1619        # Wait for the destroy to complete. 
     1620        sleep (45); 
     1621     
     1622        # Test if the operations worked. 
     1623        is(!-f $cloneConfig, 1, "destroy()") or diag("The destroy() call failed."); 
     1624    
     1625        $clone = undef; 
     1626 
     1627        if (-f $cloneConfig) { 
     1628            # Connect to daemon as a client. 
     1629            $stub = getClientHandle(namespace => "HoneyClient::Manager::VM"); 
     1630     
     1631            # Destroy the clone VM. 
     1632            $som = $stub->destroyVM(config => $cloneConfig); 
     1633        } 
     1634    } 
     1635}; 
     1636 
     1637# Kill the child daemon, if it still exists. 
     1638HoneyClient::Manager::VM->destroy(); 
     1639 
     1640# Report any failure found. 
     1641if ($@) { 
     1642    fail($@); 
     1643} 
     1644 
     1645=end testing 
     1646 
     1647=cut 
     1648 
     1649sub destroy { 
     1650 
     1651    # Extract arguments. 
     1652    my ($self, %args) = @_; 
     1653 
     1654    # Sanity check: Make sure we've been fed an object. 
     1655    unless (ref($self)) { 
     1656        $LOG->error("Error: Function must be called in reference to a " . 
     1657                    __PACKAGE__ . "->new() object!"); 
     1658        Carp::croak "Error: Function must be called in reference to a " . 
     1659                    __PACKAGE__ . "->new() object!\n"; 
     1660    } 
     1661 
     1662    # Log resolved arguments. 
     1663    $LOG->debug(sub { 
     1664        # Make Dumper format more terse. 
     1665        $Data::Dumper::Terse = 1; 
     1666        $Data::Dumper::Indent = 0; 
     1667        Dumper(\%args); 
     1668    }); 
     1669 
     1670    # Signal firewall to deny traffic from this clone. 
     1671    $self->_denyNetwork(); 
     1672 
     1673    # Extract the VM configuration file. 
     1674    my $vmConfig = $self->{'config'}; 
     1675 
     1676    # Set the internal VM configuration to undef, in order to 
     1677    # avoid potential object DESTROY() calls. 
     1678    $self->{'config'} = undef; 
     1679     
     1680    $LOG->info("Thread ID (" . threads->tid() . "): Destroying clone VM (" . $vmConfig . ")."); 
     1681    my $som = $self->{'_vm_handle'}->destroyVM(config => $vmConfig); 
     1682 
     1683    if (!defined($som)) { 
     1684        $LOG->error("Thread ID (" . threads->tid() . "): Unable to destroy VM (" . $self->{'config'} . ")."); 
     1685        $self->_changeStatus(status => "error"); 
     1686    } else { 
     1687        $self->_changeStatus(status => "deleted"); 
    15451688    } 
    15461689 
    …… 
    16831826 
    16841827    while (scalar(%{$args{'work'}})) { 
     1828 
     1829        # Before driving, check if a work unit limit has been specified 
     1830        # and if we've exceeded that limit. 
     1831        if ((getVar(name => "work_unit_limit") > 0) && 
     1832            ($self->{'work_units_processed'} >= getVar(name => "work_unit_limit"))) { 
     1833 
     1834            $LOG->info("Thread ID (" . threads->tid() . "): (" . $self->{'name'} . ") - Work Unit Limit Reached (" . getVar(name => "work_unit_limit") . ").  Recycling clone VM."); 
     1835            $self->destroy(); 
     1836        } 
     1837 
    16851838        # Create a new clone, if the current clone is not already running. 
    16861839        if ($self->{'status'} ne "running") { 
    …… 
    17021855        eval { 
    17031856            $LOG->info("Thread ID (" . threads->tid() . "): (" . $self->{'name'} . ") - " . $self->{'driver_name'} . " - Driving To Resource: " . $currentWork); 
     1857            $self->{'work_units_processed'}++; 
    17041858            $som = $self->{'_agent_handle'}->drive(driver_name => $self->{'driver_name'}, 
    17051859                                                   parameters  => encode_base64($currentWork)); 

  • honeyclient/trunk/t/honeyclient_manager.t

    r1499 r1582  
    105105require_ok('Data::Dumper'); 
    106106use Data::Dumper; 
     107 
     108# Make sure Sys::Hostname loads. 
     109BEGIN { use_ok('Sys::Hostname') or diag("Can't load Sys::Hostname package.  Check to make sure the package library is correctly listed within the path."); } 
     110require_ok('Sys::Hostname'); 
     111use Sys::Hostname; 
     112 
     113# Make sure Sys::HostIP loads. 
     114BEGIN { use_ok('Sys::HostIP') or diag("Can't load Sys::HostIP package.  Check to make sure the package library is correctly listed within the path."); } 
     115require_ok('Sys::HostIP'); 
     116use Sys::HostIP; 
    107117} 
    108118 

  • honeyclient/trunk/t/honeyclient_manager_vm_clone.t

    r1554 r1582  
    283283                       "# Your master VM is: " . getVar(name => "master_vm_config", namespace => "HoneyClient::Manager::VM") . "\n" . 
    284284                       "#\n" . 
    285                        "# Do you want to test cloning and archiving this master VM?", "no"); 
     285                       "# Do you want to test cloning this master VM and archiving a subsequent clone?", "no"); 
    286286    if ($question =~ /^y.*/i) { 
    287287 
    …… 
    332332eval { 
    333333 
     334    my $testVMDir = dirname($testVM); 
     335 
     336    # Pretend as though no other Clone objects have been created prior 
     337    # to this point. 
     338    $HoneyClient::Manager::VM::Clone::OBJECT_COUNT = -1; 
     339     
     340    my $question; 
     341    $question = prompt("#\n" . 
     342                       "# Note: Testing real destroy operations will *ONLY* work\n" . 
     343                       "# with a fully functional master VM that has the HoneyClient code\n" . 
     344                       "# loaded upon boot-up.\n" . 
     345                       "#\n" . 
     346                       "# This test also requires that the firewall VM is registered,\n" . 
     347                       "# powered on, and operational.\n" . 
     348                       "#\n" . 
     349                       "# Your master VM is: " . getVar(name => "master_vm_config", namespace => "HoneyClient::Manager::VM") . "\n" . 
     350                       "#\n" . 
     351                       "# Do you want to test cloning this master VM and destroying a subsequent clone?", "no"); 
     352    if ($question =~ /^y.*/i) { 
     353 
     354        # Create a generic empty clone, with test state data. 
     355        my $clone = HoneyClient::Manager::VM::Clone->new(_bypass_firewall => 1); 
     356        my $cloneConfig = $clone->{config}; 
     357 
     358        # Archive the clone. 
     359        $clone->destroy(); 
     360 
     361        # Wait for the destroy to complete. 
     362        sleep (45); 
     363     
     364        # Test if the operations worked. 
     365        is(!-f $cloneConfig, 1, "destroy()") or diag("The destroy() call failed."); 
     366    
     367        $clone = undef; 
     368 
     369        if (-f $cloneConfig) { 
     370            # Connect to daemon as a client. 
     371            $stub = getClientHandle(namespace => "HoneyClient::Manager::VM"); 
     372     
     373            # Destroy the clone VM. 
     374            $som = $stub->destroyVM(config => $cloneConfig); 
     375        } 
     376    } 
     377}; 
     378 
     379# Kill the child daemon, if it still exists. 
     380HoneyClient::Manager::VM->destroy(); 
     381 
     382# Report any failure found. 
     383if ($@) { 
     384    fail($@); 
     385} 
     386} 
     387 
     388 
     389 
     390# =begin testing 
     391{ 
     392# Shared test variables. 
     393my ($stub, $som, $URL); 
     394my $testVM = $ENV{PWD} . "/" . getVar(name      => "test_vm_config", 
     395                                      namespace => "HoneyClient::Manager::VM::Test"); 
     396 
     397# Catch all errors, in order to make sure child processes are 
     398# properly killed. 
     399eval { 
     400 
    334401    # Pretend as though no other Clone objects have been created prior 
    335402    # to this point.