Changeset 1547

Timestamp:

04/22/08 02:22:00 (3 weeks ago)

Author:

kindlund

Message:

Added initial support for managing simultaneous honeyclient VMs on a single Manager. Command-line functionality is complete; however, Database functionality is NOT complete. Initial stress tests and unit tests seem to pass ok. Had to re-work some of the VM daemon functionality, in order for multiple threads to play nice. ithreads is still very annoying to use — 80% of the time it works ok, but the remaining 20% of the time seems to cause the code to run into esoteric corner cases (e.g., initialization/destruction logic) that takes LONG amounts of time to debug.

Files:

• honeyclient/trunk/etc/honeyclient.xml (modified) (1 diff)
• honeyclient/trunk/lib/HoneyClient/Manager.pm (modified) (7 diffs)
• honeyclient/trunk/lib/HoneyClient/Manager/VM.pm (modified) (13 diffs)
• honeyclient/trunk/lib/HoneyClient/Manager/VM/Clone.pm (modified) (38 diffs)

honeyclient/trunk/etc/honeyclient.xml

@@ -178 +178 @@
-    </Crawler>
-    <Manager>
+        <num_simultaneous_clones description="Upon startup, the Manager will attempt to spawn the specified number of cloned VMs simultaneously.  To figure out an optimal value, take the amount of RAM allocated for your master VM (e.g., 256MB) and multiply that amount by this specified number (e.g., 4 simultaneous VMs x 256MB = 1GB).  You want to make sure that the total RAM requirements do not exceed the amount of RAM the host system can provision.  Allow the host system at least 512MB of RAM for normal operations." default="1">
+            1
+        </num_simultaneous_clones>
-        <fingerprint_dump description="When the Manager encounters a suspicious VM, it will attempt to append a complete copy of the fingerprint information to this file, if specified.">
-            fingerprint.dump

honeyclient/trunk/lib/HoneyClient/Manager.pm

@@ -287 +287 @@
-our $LOG = get_logger();
-
+# The global work queue.  Each entry represents
+# work destined for a child thread worker.
+our $WORK_QUEUE : shared = undef;
+
+# The global wait queue.  Each entry represents
+# a thread ID waiting for more work.
+our $WAIT_QUEUE : shared = undef;
+
-#######################################################################
-# Daemon Initialization / Destruction                                 #
@@ -418 +426 @@
-    };
-
+    # Verify all sub threads are finished, prior to shutting down.
+    my $thread;
+    foreach $thread (threads->list()) {
+        # Don't kill/detach the main thread or ourselves.
+        if ($thread->tid() && !$thread->equal(threads->self())) {
+            # Kill the child thread, if it's running.
+            if ($thread->is_running()) {
+                # Send empty work, if need be.
+                my $work = {};
+                $WORK_QUEUE->enqueue(nfreeze($work));
+
+                $LOG->info("Shutting down Thread ID (" . $thread->tid() . ").");
+                $thread->kill('USR1');
+
+                # Join the child thread.
+                if (!$thread->is_detached()) {
+                    $LOG->debug("Joining Thread ID (" . $thread->tid() . ").");
+                    $thread->join();
+                }
+            }
+        }
+    }
+
-    # XXX: There is an issue where if we try to quit but are in the
-    # process of asynchronously archiving a VM, then the async archive
@@ -425 +456 @@
-    kill("KILL", -$$);
-}
+
+# Helper function designed to "pop" a (key, value) pair off a given hashtable.
+# When given a hashtable reference, this function will extract a valid (key, value)
+# pair from the hashtable and delete the (key, value) pair from the
+# hashtable.  The (key, value) pair with the highest score is returned.
+#
+# Inputs: hashref
+# Outputs: valid (key, val) hashref or undef if the hash is empty
+sub _pop {
+
+    # Get supplied hash reference.
+    my $hash = shift;
+
+    # Get the highest score.
+    my @array = sort {$$hash{$b} <=> $$hash{$a}} keys %{$hash};
+    my $topkey = $array[0];
+
+    if (defined($topkey)) {
+        # Delete the key from the hashtable.
+        # Return the (key, val) pair found.
+        return { $topkey => delete($hash->{$topkey}) };
+    } else {
+        return undef;
+    }
+}
+
+# Helper function designed to take in specified work
+# and split up that work evenly amongst the specified
+# number of clones by creating individual "buckets"
+# for each clone, adding non-empty buckets to the
+# work queue.
+#
+# Inputs: work_queue, wait_queue, work, num_simultaneous_clones
+# Outputs: None.
+sub _divide_work {
+
+    # Extract arguments.
+    my (%args) = @_;
+
+    # Initalize buckets - 1 bucket per client.
+    my $buckets = [ ];
+    for (my $counter = 0; $counter < $args{'num_simultaneous_clones'}; $counter++) {
+        $buckets->[$counter] = { };
+    }
+
+    # Divide up the work evenly across each bucket.
+    while (scalar(%{$args{'work'}})) {
+        for (my $counter = 0; $counter < $args{'num_simultaneous_clones'}; $counter++) {
+            my $item = _pop($args{'work'});
+            if (defined($item)) {
+                $buckets->[$counter] = { %{$buckets->[$counter]}, %{$item} };
+            } else {
+                # We have no more work to place into any bucket,
+                # so exit the loop.
+                last;
+            }
+        }
+    }
+
+    # Place the contents of each bucket onto the work queue,
+    # but ignore empty buckets.
+    my $tid = undef;
+    for (my $counter = 0; $counter < $args{'num_simultaneous_clones'}; $counter++) {
+        if (scalar %{$buckets->[$counter]}) {
+
+            # For each bucket inserted into the work queue, delete a corresponding
+            # signal from the wait queue -- if there is a signal on the wait queue.
+            if (defined($tid = $args{'wait_queue'}->dequeue_nb)) {
+        
+                # Sanity check: Make sure the thread is still alive.
+                if (!defined(threads->object($tid))) {
+                    $LOG->error("Thread ID (" . $tid . "): Unexpectedly terminated.");
+                    Carp::croak "Thread ID (" . $tid . "): Unexpectedly terminated.";
+                }
+
+            }
+
+            # TODO: Delete this, eventually.
+            print "\nBucket #" . $counter . ":\n" . Dumper($buckets->[$counter]) . "\n\n";
+            $args{'work_queue'}->enqueue(nfreeze($buckets->[$counter]));
+        }
+    }
+}
+
+
+# Signal handler to help give user immediate feedback during
+# shutdown process.
+sub _shutdown {
+    my $LOG = get_logger();
+    $LOG->warn("Received termination signal.  Shutting down (please wait).");
+    exit;
+};
+$SIG{HUP}  = \&_shutdown;
+$SIG{INT}  = \&_shutdown;
+$SIG{QUIT} = \&_shutdown;
+$SIG{ABRT} = \&_shutdown;
+$SIG{TERM} = \&_shutdown;
-
-#######################################################################
@@ -496 +624 @@
-    }
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-    # If supported, get a URL list from the database.
-    my $remoteLinksExist = 0;
-    my $queue_url_list = {};
-
+
+
+
-        $LOG->info("Waiting for new URLs from database.");
-
+
+
+
-        $remoteLinksExist = scalar(%{$queue_url_list});
-        while (!$localLinksExist && !$remoteLinksExist) {
@@ -512 +661 @@
-            # XXX: Trap/ignore all errors and simply retry.
-            eval {
-
+
+
+
-                $remoteLinksExist = scalar(%{$queue_url_list});
-            };
@@ -522 +673 @@
-        }
-        
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        # Once finished, empty the work queue.
-        $args{'work'} = {};
@@ -535 +709 @@
-    # that was provided from the command line and then shut down.
-    if (scalar(%{$args{'work'}})) {
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        # Once finished, empty the work queue.
-        $args{'work'} = {};
-    }
-    $LOG->info("All URLs exhausted. Shutting down Manager.");
+}
+
+# TODO: Comment this.
+# Note: We can gracefully stop each worker, by sending an empty hashtable of work,
+# or by signalling the thread.
+sub _worker {
+
+    # Make sure the thread can only kill itself and not the entire application.
+    threads->set_thread_exit_only(1);
+
+    # Register interrupt/kill signal handlers.
+    # These handlers are designed to kill this thread upon overall module
+    # destruction.  These handlers should never be used for normal program
+    # operations, since they will NOT release any locks/semaphores properly.
+    local $SIG{USR1} = sub {
+        my $LOG = get_logger();
+        $LOG->warn("Thread ID (" . threads->tid() . "): Received SIGUSR1. Shutting down worker.");
+        threads->exit();
+    };
+
+    local $SIG{INT} = sub { 
+        my $LOG = get_logger();
+        $LOG->warn("Thread ID (" . threads->tid() . "): Received SIGINT. Shutting down worker.");
+        threads->exit();
+    };
+
+    # Extract arguments.
+    my $args = shift;
+
+    $LOG->info("Thread ID (" . threads->tid() . "): Starting worker.");
+
+    # Yield processing to parent thread.
+    threads->yield();
+
+    eval {
+        # Create a new cloned VM.
+        my $vm = HoneyClient::Manager::VM::Clone->new(%{$args});
+
+        # Variable to hold our work.
+        my $work = undef;
+        my $data = undef;
+
+        # If there's no work on the queue, signal that we need more work.
+        if (!$WORK_QUEUE->pending) {
+            $LOG->info("Thread ID (" . threads->tid() . "): Signaling for more work.");
+            # Signal that we're ready for more work.
+            $WAIT_QUEUE->enqueue(threads->tid());
+        }
+
+        # This is a little hackish, since calling Thread::Queue->dequeue
+        # doesn't properly handle signals.
+        $LOG->info("Thread ID (" . threads->tid() . "): Waiting for more work.");
+        while (!defined($data = $WORK_QUEUE->dequeue_nb)) {
+            # Poll the wait queue every 2 seconds.
+            # This time delay should be short.
+            threads->yield();
+            sleep(2);
+        }
+        $work = thaw($data);
+        
+        while (scalar(%{$work})) {
+            $vm = $vm->drive(work => $work);
+
+            # If there's no work on the queue, signal that we need more work.
+            if (!$WORK_QUEUE->pending) {
+                $LOG->info("Thread ID (" . threads->tid() . "): Signaling for more work.");
+                # Signal that we're ready for more work.
+                $WAIT_QUEUE->enqueue(threads->tid());
+            }
+            # This is a little hackish, since calling Thread::Queue->dequeue
+            # doesn't properly handle signals.
+            $LOG->info("Thread ID (" . threads->tid() . "): Waiting for more work.");
+            while (!defined($data = $WORK_QUEUE->dequeue_nb)) {
+                # Poll the wait queue every 2 seconds.
+                # This time delay should be short.
+                threads->yield();
+                sleep(2);
+            }
+            $work = thaw($data);
+        }
+    };
+    # Report when a fault occurs.
+    if ($@) {
+        $LOG->warn("Thread ID (" . threads->tid() . "): Encountered an error. Shutting down worker. " . $@);
+    } else {
+        $LOG->info("Thread ID (" . threads->tid() . "): Received empty work. Shutting down worker.");
+    }
+
+    # Signal to the parent that we're shutting down.
+    if (!threads->is_detached()) {
+        threads->detach();
+    }
+    $WAIT_QUEUE->enqueue(threads->tid());
+   
+    # Shut thread down.
+    threads->exit();
-}
-

honeyclient/trunk/lib/HoneyClient/Manager/VM.pm

@@ -613 +613 @@
-# used to guarantee only one operation per VM is performed
-# at any given time.
-
+ : shared
-
-# Global semaphore, designed to limit exclusive access
@@ -2304 +2304 @@
-            $vmSemaphore->up();
-            $maxThreadSemaphore->up();
-return
+threads->exit()
-        };
-
@@ -2363 +2363 @@
-        
-        $maxThreadSemaphore->up();
-return
+threads->exit()
-    };
-
@@ -4101 +4101 @@
-#            $vmSemaphore->up();
-#            $maxThreadSemaphore->up();
-return
+threads->exit()
-#        };
-#
@@ -4170 +4170 @@
-#
-#        $maxThreadSemaphore->up();
-return
+threads->exit()
-#    };
-
@@ -4378 +4378 @@
-
-    # Perform the snapshot operations...
-
+
+
+
+
+
+
+
+
+
+
+
+
+
-    my $thread = async {
-
@@ -4399 +4411 @@
-            $vmSemaphore->up();
-            $maxThreadSemaphore->up();
-return
+threads->exit()
-        };
-
@@ -4407 +4419 @@
-        # Yes, this is annoying and ugly.
-        eval {
-
-            if ($powerState != VM_EXECUTION_STATE_SUSPENDED &&
-                $powerState != VM_EXECUTION_STATE_OFF) {
-        
-                # Okay, the VM is alive; so suspend it...
-                _callback($class, "suspendVM", %args);
-            }
-
-            # Now, temporarily unregister the VM, in order to ensure
-            # it doesn't get used while we snapshot it.
-            _callback($class, "unregisterVM", %args);
-
-            # Lock the VM.
@@ -4432 +4433 @@
-            chdir $pwd;
-
-            # Unlock chdirSemaphore.
-            $chdirSemaphore->up();
-
-            if (system(getVar(name => "bin_tar"), '-C', $parentDir, '-zcpf', $args{'snapshot_file'}, @fileList) != 0) {
-                # Unlock VM, if fail.
+                $chdirSemaphore->up();
-                $vmSemaphore->up();
-                $LOG->warn("Could not snapshot VM to ($args{'snapshot_file'}). " .
@@ -4446 +4446 @@
-                                             'err');
-            }
+
+            # Unlock chdirSemaphore.
+            $chdirSemaphore->up();
-            
-            # Unlock the VM.
@@ -4465 +4468 @@
-        }
-        $maxThreadSemaphore->up();
-return
+threads->exit()
-    };
-
@@ -4707 +4710 @@
-            $LOG->warn("Asynchronous revert of ($vmDir) interrupted!");
-            $maxThreadSemaphore->up();
-return
+threads->exit()
-        };
-
@@ -4764 +4767 @@
-        }
-        $maxThreadSemaphore->up();
-return
+threads->exit()
-    };
-

honeyclient/trunk/lib/HoneyClient/Manager/VM/Clone.pm

@@ -303 +303 @@
-use threads::shared;
-
+# Make sure Thread::Semaphore loads.
+BEGIN { use_ok('Thread::Semaphore') or diag("Can't load Thread::Semaphore package.  Check to make sure the package library is correctly listed within the path."); }
+require_ok('Thread::Semaphore');
+use Thread::Semaphore;
+
-# Make sure File::Basename loads.
-BEGIN { use_ok('File::Basename', qw(dirname basename)) or diag("Can't load File::Basename package.  Check to make sure the package library is correctly listed within the path."); }
@@ -345 +350 @@
-use threads;
-use threads::shared;
+use Thread::Semaphore;
-
-# Include Global Configuration Processing Library
@@ -386 +392 @@
-# Clone objects that have been created.
-our $OBJECT_COUNT : shared = -1;
+
+# Global semaphore, designed to guarantee only one
+# thread may set/access the $OBJECT_COUNT variable
+# at any given time.
+our $OBJECT_COUNT_SEMAPHORE = Thread::Semaphore->new(1);
-
-# Include Database Libraries
@@ -559 +570 @@
-                                                fault_handler => sub { die "ERROR"; });
-
-
+Thread ID (" . threads->tid() . "): 
-        my $som = undef;
-        eval {
@@ -568 +579 @@
-        # a CTRL-C or other forced termination.
-        unless(defined($som)) {
+            # XXX: There may be an issue with multiple threads attempting
+            # to all run DESTROY simultaneously.  Worst case scenario should
+            # be a warning indicating that the VM daemon bound address is
+            # already in use.
+
-            # Make sure the VM daemon was properly destroyed.
-            HoneyClient::Manager::VM->destroy();
@@ -582 +598 @@
-
-        if (!defined($som)) {
-
+Thread ID (" . threads->tid() . "): 
-            $self->_changeStatus(status => "error");
-        } else {
@@ -596 +612 @@
-    # Upon termination, destroy the global instance of the VM manager.
-    if ($OBJECT_COUNT == 0) {
-
+Thread ID (" . threads->tid() . "): 
-        HoneyClient::Manager::VM->destroy();
-    }
@@ -622 +638 @@
-    
-    if ($errMsg !~ /Unable to flush entries in chain/) {
-
+Thread ID (" . threads->tid() . "): 
-        Carp::carp __PACKAGE__ . "->_handleFWFault(): Error occurred during processing.\n" . $errMsg;
-    }
@@ -651 +667 @@
-    if (($errMsg !~ /Connection refused/) &&
-        ($errMsg !~ /No route to host/)) {
-
+Thread ID (" . threads->tid() . "): 
-        Carp::carp __PACKAGE__ . "->_handleAgentFault(): Error occurred during processing.\n" . $errMsg;
-    }
@@ -690 +706 @@
-    # perform the quick clone operation.
-    if (!defined($self->{'config'})) {
-
+Thread ID (" . threads->tid() . "): 
-        $som = $self->{'_vm_handle'}->quickCloneVM(src_config => $self->{'master_vm_config'});
-        $ret = $som->result();
-        if (!$ret) {
-
+Thread ID (" . threads->tid() . "): 
-            Carp::croak "Unable to quick clone master VM (" . $self->{'master_vm_config'} . ").";
-        }
@@ -700 +716 @@
-        $self->{'config'} = $ret;
-    } else {
-
+Thread ID (" . threads->tid() . "): 
-        $som = $self->{'_vm_handle'}->startVM(config => $self->{'config'});
-        $ret = $som->result();
-        if (!$ret) {
-
+Thread ID (" . threads->tid() . "): 
-            Carp::croak "Unable to start clone VM (" . $self->{'config'} . ").";
-        }
@@ -711 +727 @@
-
-    # Wait until the VM gets registered, before proceeding.
-
+Thread ID (" . threads->tid() . "): 
-    $ret = undef;
-    while (!defined($ret) or !$ret) {
@@ -725 +741 @@
-
-    # Once registered, check if the VM is ON yet.
-
+Thread ID (" . threads->tid() . "): 
-    $ret = undef;
-    while (!defined($ret) or ($ret != VM_EXECUTION_STATE_ON)) {
@@ -739 +755 @@
-
-    # Now, get the VM's MAC address.
-
+Thread ID (" . threads->tid() . "): 
-    $som = $self->{'_vm_handle'}->getMACaddrVM(config => $self->{'config'});
-    $self->{'mac_address'} = $som->result();
-
-    # Now, get the VM's name.
-
+Thread ID (" . threads->tid() . "): 
-    $som = $self->{'_vm_handle'}->getNameVM(config => $self->{'config'});
-    $self->{'name'} = $som->result();
-
-    # Now, get the VM's IP address.
-
+Thread ID (" . threads->tid() . "): 
-    $ret = undef;
-    my $logMsgPrinted = 0;
@@ -761 +777 @@
-            next; # skip further processing
-        } elsif (!$logMsgPrinted) {
-
+Thread ID (" . threads->tid() . "): 
-                       $self->{'ip_address'} . ") and MAC (" . $self->{'mac_address'} . ").");
-
@@ -767 +783 @@
-            $self->_allowNetwork();
-
-
+Thread ID (" . threads->tid() . "): 
-            $logMsgPrinted = 1;
-        }
@@ -822 +838 @@
-# When given a hashtable reference, this function will extract a valid key
-# from the hashtable and delete the (key, value) pair from the
-link
+key
-#
-# Inputs: hashref
@@ -871 +887 @@
-
-        # Croak if no valid argument is supplied.
-
+Thread ID (" . threads->tid() . "): 
-        Carp::croak "Error: No status argument supplied.";
-    }
@@ -898 +914 @@
-
-                    # Warn if no valid fingerprint is supplied.
-
+Thread ID (" . threads->tid() . "): 
-                    Carp::carp __PACKAGE__ . "->_changeStatus(): (" . $self->{'name'} . ") - No valid fingerprint found.";
-
@@ -912 +928 @@
-                    # Mark the VM as suspicious indirectly, by inserting the fingerprint.
-
-
+Thread ID (" . threads->tid() . "): 
-                    # Make sure the fingerprint contains a client_id.
-                    $args{'fingerprint'}->{'client_id'} = $self->{'database_id'};
@@ -920 +936 @@
-                    };
-                    if ($@ || ($fingerprint_id == 0) || !defined($fingerprint_id)) {
-
+Thread ID (" . threads->tid() . "): 
-                    } else {
-
+Thread ID (" . threads->tid() . "): 
-                    }
-                }
@@ -952 +968 @@
-    if (length($COMPROMISE_FILE) > 0 &&
-        defined($fingerprint)) {
-
+Thread ID (" . threads->tid() . "): 
-        my $dump_file = new IO::File($COMPROMISE_FILE, "a");
-
@@ -998 +1014 @@
-    };
-
-
+Thread ID (" . threads->tid() . "): 
-    # XXX: Currently, faults get propagated -- is this okay?
-    $self->{'_fw_handle'}->addChain($netTable);
@@ -1044 +1060 @@
-    };
-
-
+Thread ID (" . threads->tid() . "): 
-    # XXX: Currently, faults get propagated -- is this okay?
-    $self->{'_fw_handle'}->deleteRules($netTable);
@@ -1066 +1082 @@
-
-    if ($datastore_attr->{bavail} < $min_space_free) {
-
+Thread ID (" . threads->tid() . "): 
-    } elsif ($snapshot_attr->{bavail} < $min_space_free) {
-
+Thread ID (" . threads->tid() . "): 
-    } else {
-        return;
-    }
-
+Thread ID (" . threads->tid() . "): 
-    exit;
-}
@@ -1296 +1312 @@
-    # Now, assign our object the appropriate namespace.
-    bless $self, $class;
+    
+    # Lock access to the $OBJECT_COUNT variable.
+    $OBJECT_COUNT_SEMAPHORE->down();
-
-    # Upon first use, start up a global instance of the VM manager.
@@ -1302 +1321 @@
-        $OBJECT_COUNT = 0;
-    }
+    
+    # Unlock access to the $OBJECT_COUNT variable.
+    $OBJECT_COUNT_SEMAPHORE->up();
-
-    # Set a valid handle for the VM daemon.
@@ -1313 +1335 @@
-    my $caller = caller();
-    if (($caller ne __PACKAGE__) && ($caller ne "HoneyClient::Manager")) {
-
+Thread ID (" . threads->tid() . "): 
-        # XXX: Currently, faults get propagated -- is this okay?
-        $self->{'_fw_handle'}->installDefaultRules();
@@ -1327 +1349 @@
-    # set the master VM to prepare for cloning.
-    unless (defined($self->{'config'})) {
-
+Thread ID (" . threads->tid() . "): 
-        my $som = $self->{'_vm_handle'}->setMasterVM(config => $self->{'master_vm_config'});
-        if (!$som->result()) {
-
+Thread ID (" . threads->tid() . "): 
-            Carp::croak "Unable to set VM (" . $self->{'master_vm_config'} . ") as a master VM.";
-        }
-    }
+    
+    # Lock access to the $OBJECT_COUNT variable.
+    $OBJECT_COUNT_SEMAPHORE->down();
-
-    # Update our global object count.
-    $OBJECT_COUNT++;
+    
+    # Unlock access to the $OBJECT_COUNT variable.
+    $OBJECT_COUNT_SEMAPHORE->up();
-
-    # Finally, return the blessed object, with a fully initialized
@@ -1493 +1521 @@
-    $self->{'config'} = undef;
-    
-
+Thread ID (" . threads->tid() . "): 
-    my $som = $self->{'_vm_handle'}->suspendVM(config => $vmConfig);
-
-    if (!defined($som)) {
-
+Thread ID (" . threads->tid() . "): 
-        $self->_changeStatus(status => "error");
-    } else {
@@ -1513 +1541 @@
-        }
-        if (!defined($som)) {
-
+Thread ID (" . threads->tid() . "): 
-        }
-    }
@@ -1639 +1667 @@
-
-        # Croak if no valid argument is supplied.
-
+Thread ID (" . threads->tid() . "): 
-        Carp::croak "Error: No work argument supplied.";
-    }
@@ -1673 +1701 @@
-        # Drive the Agent.
-        eval {
-
+Thread ID (" . threads->tid() . "): 
-            $som = $self->{'_agent_handle'}->drive(driver_name => $self->{'driver_name'},
-                                                   parameters  => encode_base64($currentWork));
@@ -1681 +1709 @@
-            # We lost communications with the Agent; assume the worst
-            # and mark the VM as suspicious.
-
+Thread ID (" . threads->tid() . "): 
-
-            # Suspend and archive the cloned VM.
@@ -1693 +1721 @@
-            if (defined($self->{'database_id'})) {
-                $numWorkInserted = HoneyClient::Manager::Database::insert_history_urls($finishedWork);
-
+"Thread ID (" . threads->tid() . "): " . 
-            }
-
@@ -1701 +1729 @@
-        # Figure out if there was a compromise found.
-        } elsif (scalar(@{$result->{'fingerprint'}->{os_processes}})) {
-
+Thread ID (" . threads->tid() . "): 
-
-            # Dump the fingerprint to a file, if need be.
@@ -1713 +1741 @@
-            if (defined($self->{'database_id'})) {
-                $numWorkInserted = HoneyClient::Manager::Database::insert_history_urls($finishedWork);
-
+"Thread ID (" . threads->tid() . "): " . 
-            }
-
@@ -1720 +1748 @@
-
-        } else {
-
+Thread ID (" . threads->tid() . "): 
-            # If possibile, insert work history.
-            $finishedWork->{'links_visited'}->{$currentWork} = $result->{'time_at'};
-            if (defined($self->{'database_id'})) {
-                $numWorkInserted = HoneyClient::Manager::Database::insert_history_urls($finishedWork);
-
+"Thread ID (" . threads->tid() . "): " . 
-            }
-        }