Changeset 105

Timestamp:

12/11/06 23:27:43 (2 years ago)

Author:

kindlund

Message:

Updated parsing logic. Ran into issue where including the Util::Config library slowed down parsing by a factor of 10x.

Files:

• honeyclient/branches/bug/42/lib/HoneyClient/Agent/Integrity/Registry/Parser.pm (modified) (23 diffs)
• honeyclient/branches/bug/42/lib/HoneyClient/Agent/Integrity/Registry/Parser.yp (modified) (16 diffs)

honeyclient/branches/bug/42/lib/HoneyClient/Agent/Integrity/Registry/Parser.pm

@@ -102 +102 @@
-use Carp ();
-
-# Include Global Configuration Processing Library
-use HoneyClient::Util::Config qw(getVar);
-
-# Include Logging Library
-use Log::Log4perl qw(:easy);
+# Temporarily Initialize Logging Subsystem
+# XXX: We hard code the logging format here, since it appears that
+#      calling 'use HoneyClient::Util::Config qw(getVar);' slows down
+#      the parser by a factor of 10x (not sure why).
+Log::Log4perl->init_once({
+    "log4perl.rootLogger"                               => "INFO, Screen",
+    "log4perl.appender.Screen"                          => "Log::Log4perl::Appender::Screen",
+    "log4perl.appender.Screen.stderr"                   => 0,
+    "log4perl.appender.Screen.Threshold"                => "INFO",
+    "log4perl.appender.Screen.layout"                   => "Log::Log4perl::Layout::PatternLayout",
+    "log4perl.appender.Screen.layout.ConversionPattern" => "%d{yyyy-MM-dd HH:mm:ss} %5p [%M] (%F:%L) - %m%n",
+});
+
-
-# Use Dumper Library.
@@ -116 +126 @@
-# Use Seek Library.
-use Fcntl qw(:seek);
+
+# Use Binary Search Library.
+use Search::Binary;
+
+# Use Progress Bar Library.
+use Term::ProgressBar;
-
-#######################################################################
@@ -208 +224 @@
-require_ok('Fcntl');
-use Fcntl qw(:seek);
+
+# Make sure Search::Binary loads
+BEGIN { use_ok('Search::Binary')
+        or diag("Can't load Search::Binary package. Check to make sure the package library is correctly listed within the path."); }
+require_ok('Search::Binary');
+can_ok('Search::Binary', 'binary_search');
+use Search::Binary;
+
+# Make sure Term::ProgressBar loads
+BEGIN { use_ok('Term::ProgressBar')
+        or diag("Can't load Term::ProgressBar package. Check to make sure the package library is correctly listed within the path."); }
+require_ok('Term::ProgressBar');
+use Term::ProgressBar;
-
-# Make sure HoneyClient::Agent::Integrity::Registry::Parser loads
@@ -354 +383 @@
-         'registry', 0,
-sub
-28
+57
-{
-            $LOG->debug("Reached end of input stream.");
@@ -364 +393 @@
-         'registry', 1,
-sub
-33
+62
-{
-            $LOG->debug("Reached end of input stream.");
@@ -374 +403 @@
-         'registry', 2,
-sub
-38
+67
-{
-            $LOG->debug("Reached end of input stream.");
@@ -396 +425 @@
-         'group', 2,
-sub
-55
+84
-{
-            my $ret = { };
@@ -415 +444 @@
-         'group', 1,
-sub
-69
+98
-{
-            my $ret = { };
@@ -440 +469 @@
-         'entry', 2,
-sub
-293
+322
-{
-            my $entry = {
@@ -455 +484 @@
-}
-
-03
+32
-
-
@@ -467 +496 @@
-# Outputs: (token_id, data) pair 
-sub _lexer {
-
-    # Identify NEWLINE token.
-    if ($_[0]->YYData->{DATA} =~ m/\G\n/cg) {
@@ -481 +509 @@
-        $_[0]->YYData->{'input_pos'} = $_[0]->YYData->{'input_pos'} ?
-                                       $_[0]->YYData->{'input_pos'} : 0;
+        $_[0]->YYData->{'input_pos'} = $_[0]->YYData->{'input_pos'} +
+                                       $_[0]->YYData->{'abs_offset'};
+
+        # Update progress bar, if defined.
+        if (defined($_[0]->YYData->{'progress'}) &&
+            ($_[0]->YYData->{'input_pos'} > $_[0]->YYData->{'progress_next_update'})) {
+            $_[0]->YYData->{'progress_next_update'} =
+                $_[0]->YYData->{'progress'}->update($_[0]->YYData->{'input_pos'});
+        }
-
-        # Identify DIR_NAME token.
@@ -535 +572 @@
-    if ($_[0]->YYData->{DATA} =~ m/\G(.*\n)/cg) {
-        $_[0]->YYData->{'input_pos'} = pos($_[0]->YYData->{DATA});
-\n
-\n
+
+
-    }
-    return ('', undef);
@@ -547 +584 @@
-# Outputs: None
-sub _error {
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-}
-
@@ -570 +674 @@
-    }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
-
-    $LOG->debug("Finished group index process.");
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-}
-
@@ -618 +721 @@
-The following functions have been implemented by any Parser object.
-
-
+
+
+
-
-=over 4
@@ -630 +735 @@
-ahead and scan the entire file, indexing the file offsets of where groups start and
-end.  Otherwise, this indexing process is not performed.
+ B<$progress> is an optional parameter.  1 specifies that the parser should display
+a progress bar, as it scans through a specified file.  Otherwise, a progress bar
+is not displayed.
- 
-I<Output>: The instantiated Parser B<$object>, fully initialized.
@@ -665 +773 @@
-        !exists($args{'input_file'}) ||
-        !defined($args{'input_file'})) {
-\n
-\n
+
+
-    }
-
@@ -672 +780 @@
-    my $fh = new IO::File($args{'input_file'}, "r");
-    if (!defined($fh)) {
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-    }
-
@@ -679 +797 @@
-    $parser->YYData->{'file_handle'} = $fh;
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
-
-    # Perform group indexing, if specified.
@@ -715 +806 @@
-        $args{'index_groups'}) {
-        $parser->_index();
+    } else {
+        $parser->YYData->{'group_index'} = [0, ];
-    }
-
@@ -910 +1003 @@
-    }
-
+    # Update progress bar, if defined.
+    if (defined($_[0]->YYData->{'progress'}) &&
+        ($_[0]->YYData->{'file_size'} <= $_[0]->YYData->{'progress_next_update'})) {
+
+        $_[0]->YYData->{'progress'}->update($_[0]->YYData->{'file_size'});
+    }
+
-    # Return the next group parsed.
-    return $self->YYParse(yylex   => \&_lexer,
@@ -1051 +1151 @@
-}
-
+=pod
+
+=head2 $object->seekToNearestGroup(absolute_offset => $offset)
+
+=over 4
+
+Given an absolute offset within the file, this function
+will seek the parser to the nearest group found B<before>
+the specified offset.
+
+I<Inputs>:
+ B<$offset> is an required parameter, specifying the absolute offset
+within the file to seek to.
+
+I<Outputs>: None.
+
+=back
+
+=begin testing
+
+my ($nextGroup, $expectedGroup);
+my $test_registry_file = $ENV{PWD} . "/" . getVar(name      => "registry_file",
+                                                  namespace => "HoneyClient::Agent::Integrity::Registry::Parser::Test");
+
+# Create a generic Parser object, with test state data.
+my $parser = HoneyClient::Agent::Integrity::Registry::Parser->init(input_file => $test_registry_file, index_groups => 1);
+
+$parser->seekToNearestGroup(absolute_offset => 84);
+
+# Verify Test Group #2
+$nextGroup = $parser->nextGroup();
+$expectedGroup = {
+    key     => 'HKEY_CURRENT_USER\Testing Group 2',
+    entries => [ {
+        name  => '@',
+        value => '\\"Annoying=Value\\"',
+    }, {
+        name  => '\\"Annoying=Key\\"',
+        value => 'Bar',
+    }, {
+        name  => 'Multiline',
+        value => 'This
+value spans
+multiple lines
+',
+    }, {
+        name  => 'Sane_Key',
+        value => '\\"Wierd=\\"Value',
+    }, ],
+};
+is_deeply($nextGroup, $expectedGroup, "seekToNearestGroup()") or diag("The seekToNearestGroup() call failed.");
+
+=end testing
+
+=cut
+
+sub seekToNearestGroup {
+    # Extract arguments.
+    my ($self, %args) = @_;
+
+    # Sanity check, don't continue, unless absolute_offset
+    # was provided.
+    my $argsExist = scalar(%args);
+    if (!$argsExist ||
+        !exists($args{'absolute_offset'}) ||
+        !defined($args{'absolute_offset'})) {
+        $LOG->fatal("Error: Unable to seek parser - no 'absolute_offset' specified!");
+        Carp::croak("Error: Unable to seek parser - no 'absolute_offset' specified!");
+    }
+
+    # Check to see if the 'group_index' has been initialized.
+    # We assume that if it has [0, ], then this has not been
+    # done.
+    my $numIndices = scalar(@{$self->YYData->{'group_index'}});
+    if ($numIndices < 2) {
+        $self->_index();
+    }
+    $numIndices = scalar(@{$self->YYData->{'group_index'}});
+
+    # Find the nearest index after the offset.
+    my $found_index = binary_search(0, $numIndices - 1, $args{'absolute_offset'}, \&_search, $self);
+
+    # Now, find the nearest index before the offset.
+    if ($found_index > 0) {
+        $found_index--;
+    }
+    my $found_offset = @{$self->YYData->{'group_index'}}[$found_index];
+
+    # XXX: Change this to debug, eventually.
+    $LOG->info("Seeking parser to nearest earlier group offset (" . $found_offset . ").");
+
+    # Seek the parser, to the specified offset.
+    $self->_reset($found_offset);
+}
+
-#######################################################################
-# Additional Module Documentation                                     #

honeyclient/branches/bug/42/lib/HoneyClient/Agent/Integrity/Registry/Parser.yp

@@ -85 +85 @@
-use Carp ();
-
-# Include Global Configuration Processing Library
-use HoneyClient::Util::Config qw(getVar);
-
-# Include Logging Library
-use Log::Log4perl qw(:easy);
+# Temporarily Initialize Logging Subsystem
+# XXX: We hard code the logging format here, since it appears that
+#      calling 'use HoneyClient::Util::Config qw(getVar);' slows down
+#      the parser by a factor of 10x (not sure why).
+Log::Log4perl->init_once({
+    "log4perl.rootLogger"                               => "INFO, Screen",
+    "log4perl.appender.Screen"                          => "Log::Log4perl::Appender::Screen",
+    "log4perl.appender.Screen.stderr"                   => 0,
+    "log4perl.appender.Screen.Threshold"                => "INFO",
+    "log4perl.appender.Screen.layout"                   => "Log::Log4perl::Layout::PatternLayout",
+    "log4perl.appender.Screen.layout.ConversionPattern" => "%d{yyyy-MM-dd HH:mm:ss} %5p [%M] (%F:%L) - %m%n",
+});
+
-
-# Use Dumper Library.
@@ -99 +109 @@
-# Use Seek Library.
-use Fcntl qw(:seek);
+
+# Use Binary Search Library.
+use Search::Binary;
+
+# Use Progress Bar Library.
+use Term::ProgressBar;
-
-#######################################################################
@@ -191 +207 @@
-require_ok('Fcntl');
-use Fcntl qw(:seek);
+
+# Make sure Search::Binary loads
+BEGIN { use_ok('Search::Binary')
+        or diag("Can't load Search::Binary package. Check to make sure the package library is correctly listed within the path."); }
+require_ok('Search::Binary');
+can_ok('Search::Binary', 'binary_search');
+use Search::Binary;
+
+# Make sure Term::ProgressBar loads
+BEGIN { use_ok('Term::ProgressBar')
+        or diag("Can't load Term::ProgressBar package. Check to make sure the package library is correctly listed within the path."); }
+require_ok('Term::ProgressBar');
+use Term::ProgressBar;
-
-# Make sure HoneyClient::Agent::Integrity::Registry::Parser loads
@@ -312 +341 @@
-# Outputs: (token_id, data) pair 
-sub _lexer {
-
-    # Identify NEWLINE token.
-    if ($_[0]->YYData->{DATA} =~ m/\G\n/cg) {
@@ -326 +354 @@
-        $_[0]->YYData->{'input_pos'} = $_[0]->YYData->{'input_pos'} ?
-                                       $_[0]->YYData->{'input_pos'} : 0;
+        $_[0]->YYData->{'input_pos'} = $_[0]->YYData->{'input_pos'} +
+                                       $_[0]->YYData->{'abs_offset'};
+
+        # Update progress bar, if defined.
+        if (defined($_[0]->YYData->{'progress'}) &&
+            ($_[0]->YYData->{'input_pos'} > $_[0]->YYData->{'progress_next_update'})) {
+            $_[0]->YYData->{'progress_next_update'} =
+                $_[0]->YYData->{'progress'}->update($_[0]->YYData->{'input_pos'});
+        }
-
-        # Identify DIR_NAME token.
@@ -380 +417 @@
-    if ($_[0]->YYData->{DATA} =~ m/\G(.*\n)/cg) {
-        $_[0]->YYData->{'input_pos'} = pos($_[0]->YYData->{DATA});
-\n
-\n
+
+
-    }
-    return ('', undef);
@@ -392 +429 @@
-# Outputs: None
-sub _error {
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-}
-
@@ -415 +519 @@
-    }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
-
-    $LOG->debug("Finished group index process.");
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-}
-
@@ -463 +566 @@
-The following functions have been implemented by any Parser object.
-
-
+
+
+
-
-=over 4
@@ -475 +580 @@
-ahead and scan the entire file, indexing the file offsets of where groups start and
-end.  Otherwise, this indexing process is not performed.
+ B<$progress> is an optional parameter.  1 specifies that the parser should display
+a progress bar, as it scans through a specified file.  Otherwise, a progress bar
+is not displayed.
- 
-I<Output>: The instantiated Parser B<$object>, fully initialized.
@@ -510 +618 @@
-        !exists($args{'input_file'}) ||
-        !defined($args{'input_file'})) {
-\n
-\n
+
+
-    }
-
@@ -517 +625 @@
-    my $fh = new IO::File($args{'input_file'}, "r");
-    if (!defined($fh)) {
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-    }
-
@@ -524 +642 @@
-    $parser->YYData->{'file_handle'} = $fh;
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
-
-    # Perform group indexing, if specified.
@@ -560 +651 @@
-        $args{'index_groups'}) {
-        $parser->_index();
+    } else {
+        $parser->YYData->{'group_index'} = [0, ];
-    }
-
@@ -755 +848 @@
-    }
-
+    # Update progress bar, if defined.
+    if (defined($_[0]->YYData->{'progress'}) &&
+        ($_[0]->YYData->{'file_size'} <= $_[0]->YYData->{'progress_next_update'})) {
+
+        $_[0]->YYData->{'progress'}->update($_[0]->YYData->{'file_size'});
+    }
+
-    # Return the next group parsed.
-    return $self->YYParse(yylex   => \&_lexer,
@@ -896 +996 @@
-}
-
+=pod
+
+=head2 $object->seekToNearestGroup(absolute_offset => $offset)
+
+=over 4
+
+Given an absolute offset within the file, this function
+will seek the parser to the nearest group found B<before>
+the specified offset.
+
+I<Inputs>:
+ B<$offset> is an required parameter, specifying the absolute offset
+within the file to seek to.
+
+I<Outputs>: None.
+
+=back
+
+=begin testing
+
+my ($nextGroup, $expectedGroup);
+my $test_registry_file = $ENV{PWD} . "/" . getVar(name      => "registry_file",
+                                                  namespace => "HoneyClient::Agent::Integrity::Registry::Parser::Test");
+
+# Create a generic Parser object, with test state data.
+my $parser = HoneyClient::Agent::Integrity::Registry::Parser->init(input_file => $test_registry_file, index_groups => 1);
+
+$parser->seekToNearestGroup(absolute_offset => 84);
+
+# Verify Test Group #2
+$nextGroup = $parser->nextGroup();
+$expectedGroup = {
+    key     => 'HKEY_CURRENT_USER\Testing Group 2',
+    entries => [ {
+        name  => '@',
+        value => '\\"Annoying=Value\\"',
+    }, {
+        name  => '\\"Annoying=Key\\"',
+        value => 'Bar',
+    }, {
+        name  => 'Multiline',
+        value => 'This
+value spans
+multiple lines
+',
+    }, {
+        name  => 'Sane_Key',
+        value => '\\"Wierd=\\"Value',
+    }, ],
+};
+is_deeply($nextGroup, $expectedGroup, "seekToNearestGroup()") or diag("The seekToNearestGroup() call failed.");
+
+=end testing
+
+=cut
+
+sub seekToNearestGroup {
+    # Extract arguments.
+    my ($self, %args) = @_;
+
+    # Sanity check, don't continue, unless absolute_offset
+    # was provided.
+    my $argsExist = scalar(%args);
+    if (!$argsExist ||
+        !exists($args{'absolute_offset'}) ||
+        !defined($args{'absolute_offset'})) {
+        $LOG->fatal("Error: Unable to seek parser - no 'absolute_offset' specified!");
+        Carp::croak("Error: Unable to seek parser - no 'absolute_offset' specified!");
+    }
+
+    # Check to see if the 'group_index' has been initialized.
+    # We assume that if it has [0, ], then this has not been
+    # done.
+    my $numIndices = scalar(@{$self->YYData->{'group_index'}});
+    if ($numIndices < 2) {
+        $self->_index();
+    }
+    $numIndices = scalar(@{$self->YYData->{'group_index'}});
+
+    # Find the nearest index after the offset.
+    my $found_index = binary_search(0, $numIndices - 1, $args{'absolute_offset'}, \&_search, $self);
+
+    # Now, find the nearest index before the offset.
+    if ($found_index > 0) {
+        $found_index--;
+    }
+    my $found_offset = @{$self->YYData->{'group_index'}}[$found_index];
+
+    # XXX: Change this to debug, eventually.
+    $LOG->info("Seeking parser to nearest earlier group offset (" . $found_offset . ").");
+
+    # Seek the parser, to the specified offset.
+    $self->_reset($found_offset);
+}
+
-#######################################################################
-# Additional Module Documentation                                     #