diff --git a/src/lib/dns/master_lexer.cc b/src/lib/dns/master_lexer.cc index 064e99d25d..b3a618f7d2 100644 --- a/src/lib/dns/master_lexer.cc +++ b/src/lib/dns/master_lexer.cc @@ -182,6 +182,15 @@ MasterLexer::getTotalSourceSize() const { return (total_size); } +size_t +MasterLexer::getPosition() const { + size_t position = 0; + BOOST_FOREACH(InputSourcePtr& src, impl_->sources_) { + position += src->getPosition(); + } + return (position); +} + const MasterToken& MasterLexer::getNextToken(Options options) { if (impl_->source_ == NULL) { diff --git a/src/lib/dns/master_lexer.h b/src/lib/dns/master_lexer.h index 26ffa184a3..84cc5d2363 100644 --- a/src/lib/dns/master_lexer.h +++ b/src/lib/dns/master_lexer.h @@ -459,6 +459,34 @@ public: /// \throw None size_t getTotalSourceSize() const; + /// \brief Return the position of lexer in the currently pushed sources. + /// + /// This method returns the position in terms of the number of recognized + /// characters from all sources. Roughly speaking, the position in a + /// single source is the offset from the beginning of the file or stream + /// to the current "read cursor" of the lexer, and the return value of + /// this method is the sum of the position in all the pushed sources. + /// + /// If the lexer reaches the end for each of all the pushed sources, + /// the return value should be equal to that of \c getTotalSourceSize(). + /// + /// If there is no source pushed in the lexer, it returns 0. + /// + /// The return values of this method and \c getTotalSourceSize() would + /// give the caller an idea of the progress of the lexer at the time of + /// the call. Note, however, that since it's not predictable whether + /// more sources will be pushed after the call, the progress determined + /// this way may not make much sense; it can only give an informational + /// hint of the progress. + /// + /// Note also that if a source is pushed, this method will normally return + /// a smaller number by definition. Likewise, the conceptual "read + /// cursor" would move backward after a call to \c ungetToken(), in which + /// case this method will return a smaller value, too. + /// + /// \throw None + size_t getPosition() const; + /// \brief Parse and return another token from the input. /// /// It reads a bit of the last opened source and produces another token diff --git a/src/lib/dns/tests/master_lexer_unittest.cc b/src/lib/dns/tests/master_lexer_unittest.cc index 3bd6a70228..65ca08a135 100644 --- a/src/lib/dns/tests/master_lexer_unittest.cc +++ b/src/lib/dns/tests/master_lexer_unittest.cc @@ -53,6 +53,7 @@ checkEmptySource(const MasterLexer& lexer) { EXPECT_TRUE(lexer.getSourceName().empty()); EXPECT_EQ(0, lexer.getSourceLine()); EXPECT_EQ(0, lexer.getTotalSourceSize()); + EXPECT_EQ(0, lexer.getPosition()); } TEST_F(MasterLexerTest, preOpen) { @@ -151,25 +152,31 @@ TEST_F(MasterLexerTest, noSource) { EXPECT_THROW(lexer.getNextToken(), isc::InvalidOperation); } -// Test getting some tokens +// Test getting some tokens. It also check basic behavior of getPosition(). TEST_F(MasterLexerTest, getNextToken) { ss << "\n \n\"STRING\"\n"; lexer.pushSource(ss); // First, the newline should get out. EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(1, lexer.getPosition()); // Then the whitespace, if we specify the option. EXPECT_EQ(MasterToken::INITIAL_WS, lexer.getNextToken(MasterLexer::INITIAL_WS).getType()); + EXPECT_EQ(2, lexer.getPosition()); // The newline EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(5, lexer.getPosition()); // 1st \n + 3 spaces, then 2nd \n // The (quoted) string EXPECT_EQ(MasterToken::QSTRING, lexer.getNextToken(MasterLexer::QSTRING).getType()); + EXPECT_EQ(5 + 8, lexer.getPosition()); // 8 = len("STRING') + quotes // And the end of line and file EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(5 + 8 + 1, lexer.getPosition()); // previous + 3rd \n EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType()); + EXPECT_EQ(5 + 8 + 1, lexer.getPosition()); // position doesn't change } // Test we correctly find end of file. @@ -214,20 +221,25 @@ TEST_F(MasterLexerTest, getUnbalancedString) { EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType()); } -// Test ungetting tokens works +// Test ungetting tokens works. Also check getPosition() is adjusted TEST_F(MasterLexerTest, ungetToken) { ss << "\n (\"string\"\n) more"; lexer.pushSource(ss); // Try getting the newline EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(1, lexer.getPosition()); // Return it and get again lexer.ungetToken(); + EXPECT_EQ(0, lexer.getPosition()); EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(1, lexer.getPosition()); // Get the string and return it back EXPECT_EQ(MasterToken::QSTRING, lexer.getNextToken(MasterLexer::QSTRING).getType()); + EXPECT_EQ(string("\n (\"string\"").size(), lexer.getPosition()); lexer.ungetToken(); + EXPECT_EQ(1, lexer.getPosition()); // back to just after 1st \n // But if we change the options, it honors them EXPECT_EQ(MasterToken::INITIAL_WS, lexer.getNextToken(MasterLexer::QSTRING | @@ -263,7 +275,8 @@ TEST_F(MasterLexerTest, ungetRealOptions) { } // Check the initial whitespace is found even in the first line of included -// file +// file. It also confirms getPosition() works for multiple sources, each +// of which is partially parsed. TEST_F(MasterLexerTest, includeAndInitialWS) { ss << " \n"; lexer.pushSource(ss); @@ -273,9 +286,11 @@ TEST_F(MasterLexerTest, includeAndInitialWS) { EXPECT_EQ(MasterToken::INITIAL_WS, lexer.getNextToken(MasterLexer::INITIAL_WS).getType()); + EXPECT_EQ(1, lexer.getPosition()); lexer.pushSource(ss2); EXPECT_EQ(MasterToken::INITIAL_WS, lexer.getNextToken(MasterLexer::INITIAL_WS).getType()); + EXPECT_EQ(2, lexer.getPosition()); // should be sum of position positions. } // Test only one token can be ungotten