text:
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Author: kenton@google.com (Kenton Varda)
//  Based on original Protocol Buffers design by
//  Sanjay Ghemawat, Jeff Dean, and others.

#include <limits.h>
#include <math.h>

#include <vector>

#include <google/protobuf/io/tokenizer.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>

#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/substitute.h>
#include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h>

namespace google {
namespace protobuf {
namespace io {
namespace {

// ===================================================================
// Data-Driven Test Infrastructure

// TODO(kenton):  This is copied from coded_stream_unittest.  This is
//   temporary until these fetaures are integrated into gTest itself.

// TEST_1D and TEST_2D are macros I'd eventually like to see added to
// gTest.  These macros can be used to declare tests which should be
// run multiple times, once for each item in some input array.  TEST_1D
// tests all cases in a single input array.  TEST_2D tests all
// combinations of cases from two arrays.  The arrays must be statically
// defined such that the GOOGLE_ARRAYSIZE() macro works on them.  Example:
//
// int kCases[] = {1, 2, 3, 4}
// TEST_1D(MyFixture, MyTest, kCases) {
//   EXPECT_GT(kCases_case, 0);
// }
//
// This test iterates through the numbers 1, 2, 3, and 4 and tests that
// they are all grater than zero.  In case of failure, the exact case
// which failed will be printed.  The case type must be printable using
// ostream::operator<<.

#define TEST_1D(FIXTURE, NAME, CASES)                                      \
  class FIXTURE##_##NAME##_DD : public FIXTURE {                           \
   protected:                                                              \
    template <typename CaseType>                                           \
    void DoSingleCase(const CaseType& CASES##_case);                       \
  };                                                                       \
                                                                           \
  TEST_F(FIXTURE##_##NAME##_DD, NAME) {                                    \
    for (int i = 0; i < GOOGLE_ARRAYSIZE(CASES); i++) {                           \
      SCOPED_TRACE(testing::Message()                                      \
        << #CASES " case #" << i << ": " << CASES[i]);                     \
      DoSingleCase(CASES[i]);                                              \
    }                                                                      \
  }                                                                        \
                                                                           \
  template <typename CaseType>                                             \
  void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType& CASES##_case)

#define TEST_2D(FIXTURE, NAME, CASES1, CASES2)                             \
  class FIXTURE##_##NAME##_DD : public FIXTURE {                           \
   protected:                                                              \
    template <typename CaseType1, typename CaseType2>                      \
    void DoSingleCase(const CaseType1& CASES1##_case,                      \
                      const CaseType2& CASES2##_case);                     \
  };                                                                       \
                                                                           \
  TEST_F(FIXTURE##_##NAME##_DD, NAME) {                                    \
    for (int i = 0; i < GOOGLE_ARRAYSIZE(CASES1); i++) {                          \
      for (int j = 0; j < GOOGLE_ARRAYSIZE(CASES2); j++) {                        \
        SCOPED_TRACE(testing::Message()                                    \
          << #CASES1 " case #" << i << ": " << CASES1[i] << ", "           \
          << #CASES2 " case #" << j << ": " << CASES2[j]);                 \
        DoSingleCase(CASES1[i], CASES2[j]);                                \
      }                                                                    \
    }                                                                      \
  }                                                                        \
                                                                           \
  template <typename CaseType1, typename CaseType2>                        \
  void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType1& CASES1##_case, \
                                           const CaseType2& CASES2##_case)

// -------------------------------------------------------------------

// An input stream that is basically like an ArrayInputStream but sometimes
// returns empty buffers, just to throw us off.
class TestInputStream : public ZeroCopyInputStream {
 public:
  TestInputStream(const void* data, int size, int block_size)
    : array_stream_(data, size, block_size), counter_(0) {}
  ~TestInputStream() {}

  // implements ZeroCopyInputStream ----------------------------------
  bool Next(const void** data, int* size) {
    // We'll return empty buffers starting with the first buffer, and every
    // 3 and 5 buffers after that.
    if (counter_ % 3 == 0 || counter_ % 5 == 0) {
      *data = NULL;
      *size = 0;
      ++counter_;
      return true;
    } else {
      ++counter_;
      return array_stream_.Next(data, size);
    }
  }

  void BackUp(int count)  { return array_stream_.BackUp(count); }
  bool Skip(int count)    { return array_stream_.Skip(count);   }
  int64 ByteCount() const { return array_stream_.ByteCount();   }

 private:
  ArrayInputStream array_stream_;
  int counter_;
};

// -------------------------------------------------------------------

// An error collector which simply concatenates all its errors into a big
// block of text which can be checked.
class TestErrorCollector : public ErrorCollector {
 public:
  TestErrorCollector() {}
  ~TestErrorCollector() {}

  string text_;

  // implements ErrorCollector ---------------------------------------
  void AddError(int line, int column, const string& message) {
    strings::SubstituteAndAppend(&text_, "$0:$1: $2\n",
                                 line, column, message);
  }
};

// -------------------------------------------------------------------

// We test each operation over a variety of block sizes to insure that
// we test cases where reads cross buffer boundaries as well as cases
// where they don't.  This is sort of a brute-force approach to this,
// but it's easy to write and easy to understand.
const int kBlockSizes[] = {1, 2, 3, 5, 7, 13, 32, 1024};

class TokenizerTest : public testing::Test {
 protected:
  // For easy testing.
  uint64 ParseInteger(const string& text) {
    uint64 result;
    EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result));
    return result;
  }
};

// ===================================================================

// These tests causes gcc 3.3.5 (and earlier?) to give the cryptic error:
//   "sorry, unimplemented: `method_call_expr' not supported by dump_expr"
#if !defined(__GNUC__) || __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3)

// In each test case, the entire input text should parse as a single token
// of the given type.
struct SimpleTokenCase {
  string input;
  Tokenizer::TokenType type;
};

inline std::ostream& operator<<(std::ostream& out,
                                const SimpleTokenCase& test_case) {
  return out << CEscape(test_case.input);
}

SimpleTokenCase kSimpleTokenCases[] = {
  // Test identifiers.
  { "hello",       Tokenizer::TYPE_IDENTIFIER },

  // Test integers.
  { "123",         Tokenizer::TYPE_INTEGER },
  { "0xab6",       Tokenizer::TYPE_INTEGER },
  { "0XAB6",       Tokenizer::TYPE_INTEGER },
  { "0X1234567",   Tokenizer::TYPE_INTEGER },
  { "0x89abcdef",  Tokenizer::TYPE_INTEGER },
  { "0x89ABCDEF",  Tokenizer::TYPE_INTEGER },
  { "01234567",    Tokenizer::TYPE_INTEGER },

  // Test floats.
  { "123.45",      Tokenizer::TYPE_FLOAT },
  { "1.",          Tokenizer::TYPE_FLOAT },
  { "1e3",         Tokenizer::TYPE_FLOAT },
  { "1E3",         Tokenizer::TYPE_FLOAT },
  { "1e-3",        Tokenizer::TYPE_FLOAT },
  { "1e+3",        Tokenizer::TYPE_FLOAT },
  { "1.e3",        Tokenizer::TYPE_FLOAT },
  { "1.2e3",       Tokenizer::TYPE_FLOAT },
  { ".1",          Tokenizer::TYPE_FLOAT },
  { ".1e3",        Tokenizer::TYPE_FLOAT },
  { ".1e-3",       Tokenizer::TYPE_FLOAT },
  { ".1e+3",       Tokenizer::TYPE_FLOAT },

  // Test strings.
  { "'hello'",     Tokenizer::TYPE_STRING },
  { "\"foo\"",     Tokenizer::TYPE_STRING },
  { "'a\"b'",      Tokenizer::TYPE_STRING },
  { "\"a'b\"",     Tokenizer::TYPE_STRING },
  { "'a\\'b'",     Tokenizer::TYPE_STRING },
  { "\"a\\\"b\"",  Tokenizer::TYPE_STRING },
  { "'\\xf'",      Tokenizer::TYPE_STRING },
  { "'\\0'",       Tokenizer::TYPE_STRING },

  // Test symbols.
  { "+",           Tokenizer::TYPE_SYMBOL },
  { ".",           Tokenizer::TYPE_SYMBOL },
};

TEST_2D(TokenizerTest, SimpleTokens, kSimpleTokenCases, kBlockSizes) {
  // Set up the tokenizer.
  TestInputStream input(kSimpleTokenCases_case.input.data(),
                        kSimpleTokenCases_case.input.size(),
                        kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);

  // Before Next() is called, the initial token should always be TYPE_START.
  EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
  EXPECT_EQ("", tokenizer.current().text);
  EXPECT_EQ(0, tokenizer.current().line);
  EXPECT_EQ(0, tokenizer.current().column);
  EXPECT_EQ(0, tokenizer.current().end_column);

  // Parse the token.
  ASSERT_TRUE(tokenizer.Next());

  // Check that it has the right type.
  EXPECT_EQ(kSimpleTokenCases_case.type, tokenizer.current().type);
  // Check that it contains the complete input text.
  EXPECT_EQ(kSimpleTokenCases_case.input, tokenizer.current().text);
  // Check that it is located at the beginning of the input
  EXPECT_EQ(0, tokenizer.current().line);
  EXPECT_EQ(0, tokenizer.current().column);
  EXPECT_EQ(kSimpleTokenCases_case.input.size(),
            tokenizer.current().end_column);

  // There should be no more input.
  EXPECT_FALSE(tokenizer.Next());

  // After Next() returns false, the token should have type TYPE_END.
  EXPECT_EQ(Tokenizer::TYPE_END, tokenizer.current().type);
  EXPECT_EQ("", tokenizer.current().text);
  EXPECT_EQ(0, tokenizer.current().line);
  EXPECT_EQ(kSimpleTokenCases_case.input.size(), tokenizer.current().column);
  EXPECT_EQ(kSimpleTokenCases_case.input.size(),
            tokenizer.current().end_column);

  // There should be no errors.
  EXPECT_TRUE(error_collector.text_.empty());
}

TEST_1D(TokenizerTest, FloatSuffix, kBlockSizes) {
  // Test the "allow_f_after_float" option.

  // Set up the tokenizer.
  const char* text = "1f 2.5f 6e3f 7F";
  TestInputStream input(text, strlen(text), kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);
  tokenizer.set_allow_f_after_float(true);

  // Advance through tokens and check that they are parsed as expected.
  ASSERT_TRUE(tokenizer.Next());
  EXPECT_EQ(tokenizer.current().text, "1f");
  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
  ASSERT_TRUE(tokenizer.Next());
  EXPECT_EQ(tokenizer.current().text, "2.5f");
  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
  ASSERT_TRUE(tokenizer.Next());
  EXPECT_EQ(tokenizer.current().text, "6e3f");
  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
  ASSERT_TRUE(tokenizer.Next());
  EXPECT_EQ(tokenizer.current().text, "7F");
  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);

  // There should be no more input.
  EXPECT_FALSE(tokenizer.Next());
  // There should be no errors.
  EXPECT_TRUE(error_collector.text_.empty());
}

#endif

// -------------------------------------------------------------------

// In each case, the input is parsed to produce a list of tokens.  The
// last token in "output" must have type TYPE_END.
struct MultiTokenCase {
  string input;
  Tokenizer::Token output[10];  // The compiler wants a constant array
                                // size for initialization to work.  There
                                // is no reason this can't be increased if
                                // needed.
};

inline std::ostream& operator<<(std::ostream& out,
                                const MultiTokenCase& test_case) {
  return out << CEscape(test_case.input);
}

MultiTokenCase kMultiTokenCases[] = {
  // Test empty input.
  { "", {
    { Tokenizer::TYPE_END       , ""     , 0,  0,  0 },
  }},

  // Test all token types at the same time.
  { "foo 1 1.2 + 'bar'", {
    { Tokenizer::TYPE_IDENTIFIER, "foo"  , 0,  0,  3 },
    { Tokenizer::TYPE_INTEGER   , "1"    , 0,  4,  5 },
    { Tokenizer::TYPE_FLOAT     , "1.2"  , 0,  6,  9 },
    { Tokenizer::TYPE_SYMBOL    , "+"    , 0, 10, 11 },
    { Tokenizer::TYPE_STRING    , "'bar'", 0, 12, 17 },
    { Tokenizer::TYPE_END       , ""     , 0, 17, 17 },
  }},

  // Test that consecutive symbols are parsed as separate tokens.
  { "!@+%", {
    { Tokenizer::TYPE_SYMBOL    , "!"    , 0, 0, 1 },
    { Tokenizer::TYPE_SYMBOL    , "@"    , 0, 1, 2 },
    { Tokenizer::TYPE_SYMBOL    , "+"    , 0, 2, 3 },
    { Tokenizer::TYPE_SYMBOL    , "%"    , 0, 3, 4 },
    { Tokenizer::TYPE_END       , ""     , 0, 4, 4 },
  }},

  // Test that newlines affect line numbers correctly.
  { "foo bar\nrab oof", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0, 3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 0,  4, 7 },
    { Tokenizer::TYPE_IDENTIFIER, "rab", 1,  0, 3 },
    { Tokenizer::TYPE_IDENTIFIER, "oof", 1,  4, 7 },
    { Tokenizer::TYPE_END       , ""   , 1,  7, 7 },
  }},

  // Test that tabs affect column numbers correctly.
  { "foo\tbar  \tbaz", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0,  3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 0,  8, 11 },
    { Tokenizer::TYPE_IDENTIFIER, "baz", 0, 16, 19 },
    { Tokenizer::TYPE_END       , ""   , 0, 19, 19 },
  }},

  // Test that tabs in string literals affect column numbers correctly.
  { "\"foo\tbar\" baz", {
    { Tokenizer::TYPE_STRING    , "\"foo\tbar\"", 0,  0, 12 },
    { Tokenizer::TYPE_IDENTIFIER, "baz"         , 0, 13, 16 },
    { Tokenizer::TYPE_END       , ""            , 0, 16, 16 },
  }},

  // Test that line comments are ignored.
  { "foo // This is a comment\n"
    "bar // This is another comment", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0,  3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 1,  0,  3 },
    { Tokenizer::TYPE_END       , ""   , 1, 30, 30 },
  }},

  // Test that block comments are ignored.
  { "foo /* This is a block comment */ bar", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0,  3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 34, 37 },
    { Tokenizer::TYPE_END       , ""   , 0, 37, 37 },
  }},

  // Test that sh-style comments are not ignored by default.
  { "foo # bar\n"
    "baz", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 },
    { Tokenizer::TYPE_SYMBOL    , "#"  , 0, 4, 5 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 6, 9 },
    { Tokenizer::TYPE_IDENTIFIER, "baz", 1, 0, 3 },
    { Tokenizer::TYPE_END       , ""   , 1, 3, 3 },
  }},

  // Test all whitespace chars
  { "foo\n\t\r\v\fbar", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0,  3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11, 14 },
    { Tokenizer::TYPE_END       , ""   , 1, 14, 14 },
  }},
};

TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {
  // Set up the tokenizer.
  TestInputStream input(kMultiTokenCases_case.input.data(),
                        kMultiTokenCases_case.input.size(),
                        kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);

  // Before Next() is called, the initial token should always be TYPE_START.
  EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
  EXPECT_EQ("", tokenizer.current().text);
  EXPECT_EQ(0, tokenizer.current().line);
  EXPECT_EQ(0, tokenizer.current().column);
  EXPECT_EQ(0, tokenizer.current().end_column);

  // Loop through all expected tokens.
  int i = 0;
  Tokenizer::Token token;
  do {
    token = kMultiTokenCases_case.output[i++];

    SCOPED_TRACE(testing::Message() << "Token #" << i << ": " << token.text);

    Tokenizer::Token previous = tokenizer.current();

    // Next() should only return false when it hits the end token.
    if (token.type != Tokenizer::TYPE_END) {
      ASSERT_TRUE(tokenizer.Next());
    } else {
      ASSERT_FALSE(tokenizer.Next());
    }

    // Check that the previous token is set correctly.
    EXPECT_EQ(previous.type, tokenizer.previous().type);
    EXPECT_EQ(previous.text, tokenizer.previous().text);
    EXPECT_EQ(previous.line, tokenizer.previous().line);
    EXPECT_EQ(previous.column, tokenizer.previous().column);
    EXPECT_EQ(previous.end_column, tokenizer.previous().end_column);

    // Check that the token matches the expected one.
    EXPECT_EQ(token.type, tokenizer.current().type);
    EXPECT_EQ(token.text, tokenizer.current().text);
    EXPECT_EQ(token.line, tokenizer.current().line);
    EXPECT_EQ(token.column, tokenizer.current().column);
    EXPECT_EQ(token.end_column, tokenizer.current().end_column);

  } while (token.type != Tokenizer::TYPE_END);

  // There should be no errors.
  EXPECT_TRUE(error_collector.text_.empty());
}

// This test causes gcc 3.3.5 (and earlier?) to give the cryptic error:
//   "sorry, unimplemented: `method_call_expr' not supported by dump_expr"
#if !defined(__GNUC__) || __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3)

TEST_1D(TokenizerTest, ShCommentStyle, kBlockSizes) {
  // Test the "comment_style" option.

  const char* text = "foo # bar\n"
                     "baz // qux\n"
                     "corge /* grault */\n"
                     "garply";
  const char* const kTokens[] = {"foo",  // "# bar" is ignored
                                 "baz", "/", "/", "qux",
                                 "corge", "/", "*", "grault", "*", "/",
                                 "garply"};

  // Set up the tokenizer.
  TestInputStream input(text, strlen(text), kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);
  tokenizer.set_comment_style(Tokenizer::SH_COMMENT_STYLE);

  // Advance through tokens and check that they are parsed as expected.
  for (int i = 0; i < GOOGLE_ARRAYSIZE(kTokens); i++) {
    EXPECT_TRUE(tokenizer.Next());
    EXPECT_EQ(tokenizer.current().text, kTokens[i]);
  }

  // There should be no more input.
  EXPECT_FALSE(tokenizer.Next());
  // There should be no errors.
  EXPECT_TRUE(error_collector.text_.empty());
}

#endif

// -------------------------------------------------------------------

// In each case, the input is expected to have two tokens named "prev" and
// "next" with comments in between.
struct DocCommentCase {
  string input;

  const char* prev_trailing_comments;
  const char* detached_comments[10];
  const char* next_leading_comments;
};

inline std::ostream& operator<<(std::ostream& out,
                                const DocCommentCase& test_case) {
  return out << CEscape(test_case.input);
}

DocCommentCase kDocCommentCases[] = {
  {
    "prev next",

    "",
    {},
    ""
      },

        {
      "prev /* ignored */ next",

      "",
      {},
      ""
        },

          {
        "prev // trailing comment\n"
            "next",

            " trailing comment\n",
            {},
            ""
          },

            {
          "prev\n"
              "// leading comment\n"
              "// line 2\n"
              "next",

              "",
              {},
              " leading comment\n"
              " line 2\n"
            },

              {
            "prev\n"
                "// trailing comment\n"
                "// line 2\n"
                "\n"
                "next",

                " trailing comment\n"
                " line 2\n",
                {},
                ""
              },

                {
              "prev // trailing comment\n"
                  "// leading comment\n"
                  "// line 2\n"
                  "next",

                  " trailing comment\n",
                  {},
                  " leading comment\n"
                  " line 2\n"
                },

                  {
                "prev /* trailing block comment */\n"
                    "/* leading block comment\n"
                    " * line 2\n"
                    " * line 3 */"
                    "next",

                    " trailing block comment ",
                    {},
                    " leading block comment\n"
                    " line 2\n"
                    " line 3 "
                  },

                    {
                  "prev\n"
                      "/* trailing block comment\n"
                      " * line 2\n"
                      " * line 3\n"
                      " */\n"
                      "/* leading block comment\n"
                      " * line 2\n"
                      " * line 3 */"
                      "next",

                      " trailing block comment\n"
                      " line 2\n"
                      " line 3\n",
                      {},
                      " leading block comment\n"
                      " line 2\n"
                      " line 3 "
                    },

                      {
                    "prev\n"
                        "// trailing comment\n"
                        "\n"
                        "// detached comment\n"
                        "// line 2\n"
                        "\n"
                        "// second detached comment\n"
                        "/* third detached comment\n"
                        " * line 2 */\n"
                        "// leading comment\n"
                        "next",

                        " trailing comment\n",
                        {
                      " detached comment\n"
                          " line 2\n",
                          " second detached comment\n",
                          " third detached comment\n"
                          " line 2 "
                        },
                          " leading comment\n"
                        },

                          {
                        "prev /**/\n"
                            "\n"
                            "// detached comment\n"
                            "\n"
                            "// leading comment\n"
                            "next",

                            "",
                            {
                          " detached comment\n"
                            },
                              " leading comment\n"
                            },

                              {
                            "prev /**/\n"
                                "// leading comment\n"
                                "next",

                                "",
                                {},
                                " leading comment\n"
                              },
                              };

TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
  // Set up the tokenizer.
  TestInputStream input(kDocCommentCases_case.input.data(),
                        kDocCommentCases_case.input.size(),
                        kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);

  // Set up a second tokenizer where we'll pass all NULLs to NextWithComments().
  TestInputStream input2(kDocCommentCases_case.input.data(),
                        kDocCommentCases_case.input.size(),
                        kBlockSizes_case);
  Tokenizer tokenizer2(&input2, &error_collector);

  tokenizer.Next();
  tokenizer2.Next();

  EXPECT_EQ("prev", tokenizer.current().text);
  EXPECT_EQ("prev", tokenizer2.current().text);

  string prev_trailing_comments;
  std::vector<string> detached_comments;
  string next_leading_comments;
  tokenizer.NextWithComments(&prev_trailing_comments, &detached_comments,
                             &next_leading_comments);
  tokenizer2.NextWithComments(NULL, NULL, NULL);
  EXPECT_EQ("next", tokenizer.current().text);
  EXPECT_EQ("next", tokenizer2.current().text);

  EXPECT_EQ(kDocCommentCases_case.prev_trailing_comments,
            prev_trailing_comments);

  for (int i = 0; i < detached_comments.size(); i++) {
    ASSERT_LT(i, GOOGLE_ARRAYSIZE(kDocCommentCases));
    ASSERT_TRUE(kDocCommentCases_case.detached_comments[i] != NULL);
    EXPECT_EQ(kDocCommentCases_case.detached_comments[i],
              detached_comments[i]);
  }

  // Verify that we matched all the detached comments.
  EXPECT_EQ(NULL,
      kDocCommentCases_case.detached_comments[detached_comments.size()]);

  EXPECT_EQ(kDocCommentCases_case.next_leading_comments,
            next_leading_comments);
}

// -------------------------------------------------------------------

// Test parse helpers.  It's not really worth setting up a full data-driven
// test here.
TEST_F(TokenizerTest, ParseInteger) {
  EXPECT_EQ(0, ParseInteger("0"));
  EXPECT_EQ(123, ParseInteger("123"));
  EXPECT_EQ(0xabcdef12u, ParseInteger("0xabcdef12"));
  EXPECT_EQ(0xabcdef12u, ParseInteger("0xABCDEF12"));
  EXPECT_EQ(kuint64max, ParseInteger("0xFFFFFFFFFFFFFFFF"));
  EXPECT_EQ(01234567, ParseInteger("01234567"));
  EXPECT_EQ(0X123, ParseInteger("0X123"));

  // Test invalid integers that may still be tokenized as integers.
  EXPECT_EQ(0, ParseInteger("0x"));

  uint64 i;

  // Test invalid integers that will never be tokenized as integers.
  EXPECT_FALSE(Tokenizer::ParseInteger("zxy", kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("1.2", kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("08", kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("0xg", kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("-1", kuint64max, &i));

  // Test overflows.
  EXPECT_TRUE (Tokenizer::ParseInteger("0", 0, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("1", 0, &i));
  EXPECT_TRUE (Tokenizer::ParseInteger("1", 1, &i));
  EXPECT_TRUE (Tokenizer::ParseInteger("12345", 12345, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("12346", 12345, &i));
  EXPECT_TRUE (Tokenizer::ParseInteger("0xFFFFFFFFFFFFFFFF" , kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("0x10000000000000000", kuint64max, &i));
}

TEST_F(TokenizerTest, ParseFloat) {
  EXPECT_DOUBLE_EQ(1    , Tokenizer::ParseFloat("1."));
  EXPECT_DOUBLE_EQ(1e3  , Tokenizer::ParseFloat("1e3"));
  EXPECT_DOUBLE_EQ(1e3  , Tokenizer::ParseFloat("1E3"));
  EXPECT_DOUBLE_EQ(1.5e3, Tokenizer::ParseFloat("1.5e3"));
  EXPECT_DOUBLE_EQ(.1   , Tokenizer::ParseFloat(".1"));
  EXPECT_DOUBLE_EQ(.25  , Tokenizer::ParseFloat(".25"));
  EXPECT_DOUBLE_EQ(.1e3 , Tokenizer::ParseFloat(".1e3"));
  EXPECT_DOUBLE_EQ(.25e3, Tokenizer::ParseFloat(".25e3"));
  EXPECT_DOUBLE_EQ(.1e+3, Tokenizer::ParseFloat(".1e+3"));
  EXPECT_DOUBLE_EQ(.1e-3, Tokenizer::ParseFloat(".1e-3"));
  EXPECT_DOUBLE_EQ(5    , Tokenizer::ParseFloat("5"));
  EXPECT_DOUBLE_EQ(6e-12, Tokenizer::ParseFloat("6e-12"));
  EXPECT_DOUBLE_EQ(1.2  , Tokenizer::ParseFloat("1.2"));
  EXPECT_DOUBLE_EQ(1.e2 , Tokenizer::ParseFloat("1.e2"));

  // Test invalid integers that may still be tokenized as integers.
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e"));
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e-"));
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.e"));

  // Test 'f' suffix.
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1f"));
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.0f"));
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1F"));

  // These should parse successfully even though they are out of range.
  // Overflows become infinity and underflows become zero.
  EXPECT_EQ(     0.0, Tokenizer::ParseFloat("1e-9999999999999999999999999999"));
  EXPECT_EQ(HUGE_VAL, Tokenizer::ParseFloat("1e+9999999999999999999999999999"));

#ifdef PROTOBUF_HAS_DEATH_TEST  // death tests do not work on Windows yet
  // Test invalid integers that will never be tokenized as integers.
  EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("zxy"),
    "passed text that could not have been tokenized as a float");
  EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("1-e0"),
    "passed text that could not have been tokenized as a float");
  EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("-1.0"),
    "passed text that could not have been tokenized as a float");
#endif  // PROTOBUF_HAS_DEATH_TEST
}

TEST_F(TokenizerTest, ParseString) {
  string output;
  Tokenizer::ParseString("'hello'", &output);
  EXPECT_EQ("hello", output);
  Tokenizer::ParseString("\"blah\\nblah2\"", &output);
  EXPECT_EQ("blah\nblah2", output);
  Tokenizer::ParseString("'\\1x\\1\\123\\739\\52\\334n\\3'", &output);
  EXPECT_EQ("\1x\1\123\739\52\334n\3", output);
  Tokenizer::ParseString("'\\x20\\x4'", &output);
  EXPECT_EQ("\x20\x4", output);

  // Test invalid strings that may still be tokenized as strings.
  Tokenizer::ParseString("\"\\a\\l\\v\\t", &output);  // \l is invalid
  EXPECT_EQ("\a?\v\t", output);
  Tokenizer::ParseString("'", &output);
  EXPECT_EQ("", output);
  Tokenizer::ParseString("'\\", &output);
  EXPECT_EQ("\\", output);

  // Experiment with Unicode escapes. Here are one-, two- and three-byte Unicode
  // characters.
  Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\U00024b62XX'", &output);
  EXPECT_EQ("$¢€𤭢XX", output);
  // Same thing encoded using UTF16.
  Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\ud852\\udf62XX'", &output);
  EXPECT_EQ("$¢€𤭢XX", output);
  // Here's some broken UTF16; there's a head surrogate with no tail surrogate.
  // We just output this as if it were UTF8; it's not a defined code point, but
  // it has a defined encoding.
  Tokenizer::ParseString("'\\ud852XX'", &output);
  EXPECT_EQ("\xed\xa1\x92XX", output);
  // Malformed escape: Demons may fly out of the nose.
  Tokenizer::ParseString("\\u0", &output);
  EXPECT_EQ("u0", output);

  // Test invalid strings that will never be tokenized as strings.
#ifdef PROTOBUF_HAS_DEATH_TEST  // death tests do not work on Windows yet
  EXPECT_DEBUG_DEATH(Tokenizer::ParseString("", &output),
    "passed text that could not have been tokenized as a string");
#endif  // PROTOBUF_HAS_DEATH_TEST
}

TEST_F(TokenizerTest, ParseStringAppend) {
  // Check that ParseString and ParseStringAppend differ.
  string output("stuff+");
  Tokenizer::ParseStringAppend("'hello'", &output);
  EXPECT_EQ("stuff+hello", output);
  Tokenizer::ParseString("'hello'", &output);
  EXPECT_EQ("hello", output);
}

// -------------------------------------------------------------------

// Each case parses some input text, ignoring the tokens produced, and
// checks that the error output matches what is expected.
struct ErrorCase {
  string input;
  bool recoverable;  // True if the tokenizer should be able to recover and
                     // parse more tokens after seeing this error.  Cases
                     // for which this is true must end with "foo" as
                     // the last token, which the test will check for.
  const char* errors;
};

inline std::ostream& operator<<(std::ostream& out, const ErrorCase& test_case) {
  return out << CEscape(test_case.input);
}

ErrorCase kErrorCases[] = {
  // String errors.
  { "'\\l' foo", true,
    "0:2: Invalid escape sequence in string literal.\n" },
  { "'\\X' foo", true,
    "0:2: Invalid escape sequence in string literal.\n" },
  { "'\\x' foo", true,
    "0:3: Expected hex digits for escape sequence.\n" },
  { "'foo", false,
    "0:4: Unexpected end of string.\n" },
  { "'bar\nfoo", true,
    "0:4: String literals cannot cross line boundaries.\n" },
  { "'\\u01' foo", true,
    "0:5: Expected four hex digits for \\u escape sequence.\n" },
  { "'\\u01' foo", true,
    "0:5: Expected four hex digits for \\u escape sequence.\n" },
  { "'\\uXYZ' foo", true,
    "0:3: Expected four hex digits for \\u escape sequence.\n" },

  // Integer errors.
  { "123foo", true,
    "0:3: Need space between number and identifier.\n" },

  // Hex/octal errors.
  { "0x foo", true,
    "0:2: \"0x\" must be followed by hex digits.\n" },
  { "0541823 foo", true,
    "0:4: Numbers starting with leading zero must be in octal.\n" },
  { "0x123z foo", true,
    "0:5: Need space between number and identifier.\n" },
  { "0x123.4 foo", true,
    "0:5: Hex and octal numbers must be integers.\n" },
  { "0123.4 foo", true,
    "0:4: Hex and octal numbers must be integers.\n" },

  // Float errors.
  { "1e foo", true,
    "0:2: \"e\" must be followed by exponent.\n" },
  { "1e- foo", true,
    "0:3: \"e\" must be followed by exponent.\n" },
  { "1.2.3 foo", true,
    "0:3: Already saw decimal point or exponent; can't have another one.\n" },
  { "1e2.3 foo", true,
    "0:3: Already saw decimal point or exponent; can't have another one.\n" },
  { "a.1 foo", true,
    "0:1: Need space between identifier and decimal point.\n" },
  // allow_f_after_float not enabled, so this should be an error.
  { "1.0f foo", true,
    "0:3: Need space between number and identifier.\n" },

  // Block comment errors.
  { "/*", false,
    "0:2: End-of-file inside block comment.\n"
    "0:0:   Comment started here.\n"},
  { "/*/*/ foo", true,
    "0:3: \"/*\" inside block comment.  Block comments cannot be nested.\n"},

  // Control characters.  Multiple consecutive control characters should only
  // produce one error.
  { "\b foo", true,
    "0:0: Invalid control characters encountered in text.\n" },
  { "\b\b foo", true,
    "0:0: Invalid control characters encountered in text.\n" },

  // Check that control characters at end of input don't result in an
  // infinite loop.
  { "\b", false,
    "0:0: Invalid control characters encountered in text.\n" },

  // Check recovery from '\0'.  We have to explicitly specify the length of
  // these strings because otherwise the string constructor will just call
  // strlen() which will see the first '\0' and think that is the end of the
  // string.
  { string("\0foo", 4), true,
    "0:0: Invalid control characters encountered in text.\n" },
  { string("\0\0foo", 5), true,
    "0:0: Invalid control characters encountered in text.\n" },

  // Check error from high order bits set
  { "\300foo", true,
    "0:0: Interpreting non ascii codepoint 192.\n" },
};

TEST_2D(TokenizerTest, Errors, kErrorCases, kBlockSizes) {
  // Set up the tokenizer.
  TestInputStream input(kErrorCases_case.input.data(),
                        kErrorCases_case.input.size(),
                        kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);

  // Ignore all input, except remember if the last token was "foo".
  bool last_was_foo = false;
  while (tokenizer.Next()) {
    last_was_foo = tokenizer.current().text == "foo";
  }

  // Check that the errors match what was expected.
  EXPECT_EQ(kErrorCases_case.errors, error_collector.text_);

  // If the error was recoverable, make sure we saw "foo" after it.
  if (kErrorCases_case.recoverable) {
    EXPECT_TRUE(last_was_foo);
  }
}

// -------------------------------------------------------------------

TEST_1D(TokenizerTest, BackUpOnDestruction, kBlockSizes) {
  string text = "foo bar";
  TestInputStream input(text.data(), text.size(), kBlockSizes_case);

  // Create a tokenizer, read one token, then destroy it.
  {
    TestErrorCollector error_collector;
    Tokenizer tokenizer(&input, &error_collector);

    tokenizer.Next();
  }

  // Only "foo" should have been read.
  EXPECT_EQ(strlen("foo"), input.ByteCount());
}

}  // namespace
}  // namespace io
}  // namespace protobuf
}  // namespace google


text:
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Author: kenton@google.com (Kenton Varda)
//  Based on original Protocol Buffers design by
//  Sanjay Ghemawat, Jeff Dean, and others.
//
// This implementation is heavily optimized to make reads and writes
// of small values (especially varints) as fast as possible.  In
// particular, we optimize for the common case that a read or a write
// will not cross the end of the buffer, since we can avoid a lot
// of branching in this case.

#include <google/protobuf/io/coded_stream_inl.h>
#include <algorithm>
#include <utility>
#include <limits.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/arena.h>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/stl_util.h>

namespace google {
namespace protobuf {
namespace io {

namespace {

static const int kMaxVarintBytes = 10;
static const int kMaxVarint32Bytes = 5;

inline bool NextNonEmpty(ZeroCopyInputStream* input,
                         const void** data, int* size) {
  bool success;
  do {
    success = input->Next(data, size);
  } while (success && *size == 0);
  return success;
}

}  // namespace

// CodedInputStream ==================================================

CodedInputStream::~CodedInputStream() {
  if (input_ != NULL) {
    BackUpInputToCurrentPosition();
  }
}

// Static.
int CodedInputStream::default_recursion_limit_ = 100;

void CodedOutputStream::EnableAliasing(bool enabled) {
  aliasing_enabled_ = enabled && output_->AllowsAliasing();
}

void CodedInputStream::BackUpInputToCurrentPosition() {
  int backup_bytes = BufferSize() + buffer_size_after_limit_ + overflow_bytes_;
  if (backup_bytes > 0) {
    input_->BackUp(backup_bytes);

    // total_bytes_read_ doesn't include overflow_bytes_.
    total_bytes_read_ -= BufferSize() + buffer_size_after_limit_;
    buffer_end_ = buffer_;
    buffer_size_after_limit_ = 0;
    overflow_bytes_ = 0;
  }
}

inline void CodedInputStream::RecomputeBufferLimits() {
  buffer_end_ += buffer_size_after_limit_;
  int closest_limit = std::min(current_limit_, total_bytes_limit_);
  if (closest_limit < total_bytes_read_) {
    // The limit position is in the current buffer.  We must adjust
    // the buffer size accordingly.
    buffer_size_after_limit_ = total_bytes_read_ - closest_limit;
    buffer_end_ -= buffer_size_after_limit_;
  } else {
    buffer_size_after_limit_ = 0;
  }
}

CodedInputStream::Limit CodedInputStream::PushLimit(int byte_limit) {
  // Current position relative to the beginning of the stream.
  int current_position = CurrentPosition();

  Limit old_limit = current_limit_;

  // security: byte_limit is possibly evil, so check for negative values
  // and overflow. Also check that the new requested limit is before the
  // previous limit; otherwise we continue to enforce the previous limit.
  if GOOGLE_PREDICT_TRUE(byte_limit >= 0 &&
                  byte_limit <= INT_MAX - current_position &&
                  byte_limit < current_limit_ - current_position) {
    current_limit_ = current_position + byte_limit;
    RecomputeBufferLimits();
  }

  return old_limit;
}

void CodedInputStream::PopLimit(Limit limit) {
  // The limit passed in is actually the *old* limit, which we returned from
  // PushLimit().
  current_limit_ = limit;
  RecomputeBufferLimits();

  // We may no longer be at a legitimate message end.  ReadTag() needs to be
  // called again to find out.
  legitimate_message_end_ = false;
}

std::pair<CodedInputStream::Limit, int>
CodedInputStream::IncrementRecursionDepthAndPushLimit(int byte_limit) {
  return std::make_pair(PushLimit(byte_limit), --recursion_budget_);
}

CodedInputStream::Limit CodedInputStream::ReadLengthAndPushLimit() {
  uint32 length;
  return PushLimit(ReadVarint32(&length) ? length : 0);
}

bool CodedInputStream::DecrementRecursionDepthAndPopLimit(Limit limit) {
  bool result = ConsumedEntireMessage();
  PopLimit(limit);
  GOOGLE_DCHECK_LT(recursion_budget_, recursion_limit_);
  ++recursion_budget_;
  return result;
}

bool CodedInputStream::CheckEntireMessageConsumedAndPopLimit(Limit limit) {
  bool result = ConsumedEntireMessage();
  PopLimit(limit);
  return result;
}

int CodedInputStream::BytesUntilLimit() const {
  if (current_limit_ == INT_MAX) return -1;
  int current_position = CurrentPosition();

  return current_limit_ - current_position;
}

void CodedInputStream::SetTotalBytesLimit(
    int total_bytes_limit, int warning_threshold) {
  (void) warning_threshold;

  // Make sure the limit isn't already past, since this could confuse other
  // code.
  int current_position = CurrentPosition();
  total_bytes_limit_ = std::max(current_position, total_bytes_limit);
  RecomputeBufferLimits();
}

int CodedInputStream::BytesUntilTotalBytesLimit() const {
  if (total_bytes_limit_ == INT_MAX) return -1;
  return total_bytes_limit_ - CurrentPosition();
}

void CodedInputStream::PrintTotalBytesLimitError() {
  GOOGLE_LOG(ERROR) << "A protocol message was rejected because it was too "
                "big (more than " << total_bytes_limit_
             << " bytes).  To increase the limit (or to disable these "
                "warnings), see CodedInputStream::SetTotalBytesLimit() "
                "in google/protobuf/io/coded_stream.h.";
}

bool CodedInputStream::SkipFallback(int count, int original_buffer_size) {
  if (buffer_size_after_limit_ > 0) {
    // We hit a limit inside this buffer.  Advance to the limit and fail.
    Advance(original_buffer_size);
    return false;
  }

  count -= original_buffer_size;
  buffer_ = NULL;
  buffer_end_ = buffer_;

  // Make sure this skip doesn't try to skip past the current limit.
  int closest_limit = std::min(current_limit_, total_bytes_limit_);
  int bytes_until_limit = closest_limit - total_bytes_read_;
  if (bytes_until_limit < count) {
    // We hit the limit.  Skip up to it then fail.
    if (bytes_until_limit > 0) {
      total_bytes_read_ = closest_limit;
      input_->Skip(bytes_until_limit);
    }
    return false;
  }

  total_bytes_read_ += count;
  return input_->Skip(count);
}

bool CodedInputStream::GetDirectBufferPointer(const void** data, int* size) {
  if (BufferSize() == 0 && !Refresh()) return false;

  *data = buffer_;
  *size = BufferSize();
  return true;
}

bool CodedInputStream::ReadRaw(void* buffer, int size) {
  return InternalReadRawInline(buffer, size);
}

bool CodedInputStream::ReadString(string* buffer, int size) {
  if (size < 0) return false;  // security: size is often user-supplied
  return InternalReadStringInline(buffer, size);
}

bool CodedInputStream::ReadStringFallback(string* buffer, int size) {
  if (!buffer->empty()) {
    buffer->clear();
  }

  int closest_limit = std::min(current_limit_, total_bytes_limit_);
  if (closest_limit != INT_MAX) {
    int bytes_to_limit = closest_limit - CurrentPosition();
    if (bytes_to_limit > 0 && size > 0 && size <= bytes_to_limit) {
      buffer->reserve(size);
    }
  }

  int current_buffer_size;
  while ((current_buffer_size = BufferSize()) < size) {
    // Some STL implementations "helpfully" crash on buffer->append(NULL, 0).
    if (current_buffer_size != 0) {
      // Note:  string1.append(string2) is O(string2.size()) (as opposed to
      //   O(string1.size() + string2.size()), which would be bad).
      buffer->append(reinterpret_cast<const char*>(buffer_),
                     current_buffer_size);
    }
    size -= current_buffer_size;
    Advance(current_buffer_size);
    if (!Refresh()) return false;
  }

  buffer->append(reinterpret_cast<const char*>(buffer_), size);
  Advance(size);

  return true;
}

bool CodedInputStream::ReadLittleEndian32Fallback(uint32* value) {
  uint8 bytes[sizeof(*value)];

  const uint8* ptr;
  if (BufferSize() >= sizeof(*value)) {
    // Fast path:  Enough bytes in the buffer to read directly.
    ptr = buffer_;
    Advance(sizeof(*value));
  } else {
    // Slow path:  Had to read past the end of the buffer.
    if (!ReadRaw(bytes, sizeof(*value))) return false;
    ptr = bytes;
  }
  ReadLittleEndian32FromArray(ptr, value);
  return true;
}

bool CodedInputStream::ReadLittleEndian64Fallback(uint64* value) {
  uint8 bytes[sizeof(*value)];

  const uint8* ptr;
  if (BufferSize() >= sizeof(*value)) {
    // Fast path:  Enough bytes in the buffer to read directly.
    ptr = buffer_;
    Advance(sizeof(*value));
  } else {
    // Slow path:  Had to read past the end of the buffer.
    if (!ReadRaw(bytes, sizeof(*value))) return false;
    ptr = bytes;
  }
  ReadLittleEndian64FromArray(ptr, value);
  return true;
}

namespace {

// Read a varint from the given buffer, write it to *value, and return a pair.
// The first part of the pair is true iff the read was successful.  The second
// part is buffer + (number of bytes read).  This function is always inlined,
// so returning a pair is costless.
GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
::std::pair<bool, const uint8*> ReadVarint32FromArray(
    uint32 first_byte, const uint8* buffer,
    uint32* value);
inline ::std::pair<bool, const uint8*> ReadVarint32FromArray(
    uint32 first_byte, const uint8* buffer, uint32* value) {
  // Fast path:  We have enough bytes left in the buffer to guarantee that
  // this read won't cross the end, so we can skip the checks.
  GOOGLE_DCHECK_EQ(*buffer, first_byte);
  GOOGLE_DCHECK_EQ(first_byte & 0x80, 0x80) << first_byte;
  const uint8* ptr = buffer;
  uint32 b;
  uint32 result = first_byte - 0x80;
  ++ptr;  // We just processed the first byte.  Move on to the second.
  b = *(ptr++); result += b <<  7; if (!(b & 0x80)) goto done;
  result -= 0x80 << 7;
  b = *(ptr++); result += b << 14; if (!(b & 0x80)) goto done;
  result -= 0x80 << 14;
  b = *(ptr++); result += b << 21; if (!(b & 0x80)) goto done;
  result -= 0x80 << 21;
  b = *(ptr++); result += b << 28; if (!(b & 0x80)) goto done;
  // "result -= 0x80 << 28" is irrevelant.

  // If the input is larger than 32 bits, we still need to read it all
  // and discard the high-order bits.
  for (int i = 0; i < kMaxVarintBytes - kMaxVarint32Bytes; i++) {
    b = *(ptr++); if (!(b & 0x80)) goto done;
  }

  // We have overrun the maximum size of a varint (10 bytes).  Assume
  // the data is corrupt.
  return std::make_pair(false, ptr);

 done:
  *value = result;
  return std::make_pair(true, ptr);
}

GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE::std::pair<bool, const uint8*>
ReadVarint64FromArray(const uint8* buffer, uint64* value);
inline ::std::pair<bool, const uint8*> ReadVarint64FromArray(
    const uint8* buffer, uint64* value) {
  const uint8* ptr = buffer;
  uint32 b;

  // Splitting into 32-bit pieces gives better performance on 32-bit
  // processors.
  uint32 part0 = 0, part1 = 0, part2 = 0;

  b = *(ptr++); part0  = b      ; if (!(b & 0x80)) goto done;
  part0 -= 0x80;
  b = *(ptr++); part0 += b <<  7; if (!(b & 0x80)) goto done;
  part0 -= 0x80 << 7;
  b = *(ptr++); part0 += b << 14; if (!(b & 0x80)) goto done;
  part0 -= 0x80 << 14;
  b = *(ptr++); part0 += b << 21; if (!(b & 0x80)) goto done;
  part0 -= 0x80 << 21;
  b = *(ptr++); part1  = b      ; if (!(b & 0x80)) goto done;
  part1 -= 0x80;
  b = *(ptr++); part1 += b <<  7; if (!(b & 0x80)) goto done;
  part1 -= 0x80 << 7;
  b = *(ptr++); part1 += b << 14; if (!(b & 0x80)) goto done;
  part1 -= 0x80 << 14;
  b = *(ptr++); part1 += b << 21; if (!(b & 0x80)) goto done;
  part1 -= 0x80 << 21;
  b = *(ptr++); part2  = b      ; if (!(b & 0x80)) goto done;
  part2 -= 0x80;
  b = *(ptr++); part2 += b <<  7; if (!(b & 0x80)) goto done;
  // "part2 -= 0x80 << 7" is irrelevant because (0x80 << 7) << 56 is 0.

  // We have overrun the maximum size of a varint (10 bytes).  Assume
  // the data is corrupt.
  return std::make_pair(false, ptr);

 done:
  *value = (static_cast<uint64>(part0)) |
           (static_cast<uint64>(part1) << 28) |
           (static_cast<uint64>(part2) << 56);
  return std::make_pair(true, ptr);
}

}  // namespace

bool CodedInputStream::ReadVarint32Slow(uint32* value) {
  // Directly invoke ReadVarint64Fallback, since we already tried to optimize
  // for one-byte varints.
  std::pair<uint64, bool> p = ReadVarint64Fallback();
  *value = static_cast<uint32>(p.first);
  return p.second;
}

int64 CodedInputStream::ReadVarint32Fallback(uint32 first_byte_or_zero) {
  if (BufferSize() >= kMaxVarintBytes ||
      // Optimization:  We're also safe if the buffer is non-empty and it ends
      // with a byte that would terminate a varint.
      (buffer_end_ > buffer_ && !(buffer_end_[-1] & 0x80))) {
    GOOGLE_DCHECK_NE(first_byte_or_zero, 0)
        << "Caller should provide us with *buffer_ when buffer is non-empty";
    uint32 temp;
    ::std::pair<bool, const uint8*> p =
          ReadVarint32FromArray(first_byte_or_zero, buffer_, &temp);
    if (!p.first) return -1;
    buffer_ = p.second;
    return temp;
  } else {
    // Really slow case: we will incur the cost of an extra function call here,
    // but moving this out of line reduces the size of this function, which
    // improves the common case. In micro benchmarks, this is worth about 10-15%
    uint32 temp;
    return ReadVarint32Slow(&temp) ? static_cast<int64>(temp) : -1;
  }
}

int CodedInputStream::ReadVarintSizeAsIntSlow() {
  // Directly invoke ReadVarint64Fallback, since we already tried to optimize
  // for one-byte varints.
  std::pair<uint64, bool> p = ReadVarint64Fallback();
  if (!p.second || p.first > static_cast<uint64>(INT_MAX)) return -1;
  return p.first;
}

int CodedInputStream::ReadVarintSizeAsIntFallback() {
  if (BufferSize() >= kMaxVarintBytes ||
      // Optimization:  We're also safe if the buffer is non-empty and it ends
      // with a byte that would terminate a varint.
      (buffer_end_ > buffer_ && !(buffer_end_[-1] & 0x80))) {
    uint64 temp;
    ::std::pair<bool, const uint8*> p = ReadVarint64FromArray(buffer_, &temp);
    if (!p.first || temp > static_cast<uint64>(INT_MAX)) return -1;
    buffer_ = p.second;
    return temp;
  } else {
    // Really slow case: we will incur the cost of an extra function call here,
    // but moving this out of line reduces the size of this function, which
    // improves the common case. In micro benchmarks, this is worth about 10-15%
    return ReadVarintSizeAsIntSlow();
  }
}

uint32 CodedInputStream::ReadTagSlow() {
  if (buffer_ == buffer_end_) {
    // Call refresh.
    if (!Refresh()) {
      // Refresh failed.  Make sure that it failed due to EOF, not because
      // we hit total_bytes_limit_, which, unlike normal limits, is not a
      // valid place to end a message.
      int current_position = total_bytes_read_ - buffer_size_after_limit_;
      if (current_position >= total_bytes_limit_) {
        // Hit total_bytes_limit_.  But if we also hit the normal limit,
        // we're still OK.
        legitimate_message_end_ = current_limit_ == total_bytes_limit_;
      } else {
        legitimate_message_end_ = true;
      }
      return 0;
    }
  }

  // For the slow path, just do a 64-bit read. Try to optimize for one-byte tags
  // again, since we have now refreshed the buffer.
  uint64 result = 0;
  if (!ReadVarint64(&result)) return 0;
  return static_cast<uint32>(result);
}

uint32 CodedInputStream::ReadTagFallback(uint32 first_byte_or_zero) {
  const int buf_size = BufferSize();
  if (buf_size >= kMaxVarintBytes ||
      // Optimization:  We're also safe if the buffer is non-empty and it ends
      // with a byte that would terminate a varint.
      (buf_size > 0 && !(buffer_end_[-1] & 0x80))) {
    GOOGLE_DCHECK_EQ(first_byte_or_zero, buffer_[0]);
    if (first_byte_or_zero == 0) {
      ++buffer_;
      return 0;
    }
    uint32 tag;
    ::std::pair<bool, const uint8*> p =
        ReadVarint32FromArray(first_byte_or_zero, buffer_, &tag);
    if (!p.first) {
      return 0;
    }
    buffer_ = p.second;
    return tag;
  } else {
    // We are commonly at a limit when attempting to read tags. Try to quickly
    // detect this case without making another function call.
    if ((buf_size == 0) &&
        ((buffer_size_after_limit_ > 0) ||
         (total_bytes_read_ == current_limit_)) &&
        // Make sure that the limit we hit is not total_bytes_limit_, since
        // in that case we still need to call Refresh() so that it prints an
        // error.
        total_bytes_read_ - buffer_size_after_limit_ < total_bytes_limit_) {
      // We hit a byte limit.
      legitimate_message_end_ = true;
      return 0;
    }
    return ReadTagSlow();
  }
}

bool CodedInputStream::ReadVarint64Slow(uint64* value) {
  // Slow path:  This read might cross the end of the buffer, so we
  // need to check and refresh the buffer if and when it does.

  uint64 result = 0;
  int count = 0;
  uint32 b;

  do {
    if (count == kMaxVarintBytes) {
      *value = 0;
      return false;
    }
    while (buffer_ == buffer_end_) {
      if (!Refresh()) {
        *value = 0;
        return false;
      }
    }
    b = *buffer_;
    result |= static_cast<uint64>(b & 0x7F) << (7 * count);
    Advance(1);
    ++count;
  } while (b & 0x80);

  *value = result;
  return true;
}

std::pair<uint64, bool> CodedInputStream::ReadVarint64Fallback() {
  if (BufferSize() >= kMaxVarintBytes ||
      // Optimization:  We're also safe if the buffer is non-empty and it ends
      // with a byte that would terminate a varint.
      (buffer_end_ > buffer_ && !(buffer_end_[-1] & 0x80))) {
    uint64 temp;
    ::std::pair<bool, const uint8*> p = ReadVarint64FromArray(buffer_, &temp);
    if (!p.first) {
      return std::make_pair(0, false);
    }
    buffer_ = p.second;
    return std::make_pair(temp, true);
  } else {
    uint64 temp;
    bool success = ReadVarint64Slow(&temp);
    return std::make_pair(temp, success);
  }
}

bool CodedInputStream::Refresh() {
  GOOGLE_DCHECK_EQ(0, BufferSize());

  if (buffer_size_after_limit_ > 0 || overflow_bytes_ > 0 ||
      total_bytes_read_ == current_limit_) {
    // We've hit a limit.  Stop.
    int current_position = total_bytes_read_ - buffer_size_after_limit_;

    if (current_position >= total_bytes_limit_ &&
        total_bytes_limit_ != current_limit_) {
      // Hit total_bytes_limit_.
      PrintTotalBytesLimitError();
    }

    return false;
  }

  const void* void_buffer;
  int buffer_size;
  if (NextNonEmpty(input_, &void_buffer, &buffer_size)) {
    buffer_ = reinterpret_cast<const uint8*>(void_buffer);
    buffer_end_ = buffer_ + buffer_size;
    GOOGLE_CHECK_GE(buffer_size, 0);

    if (total_bytes_read_ <= INT_MAX - buffer_size) {
      total_bytes_read_ += buffer_size;
    } else {
      // Overflow.  Reset buffer_end_ to not include the bytes beyond INT_MAX.
      // We can't get that far anyway, because total_bytes_limit_ is guaranteed
      // to be less than it.  We need to keep track of the number of bytes
      // we discarded, though, so that we can call input_->BackUp() to back
      // up over them on destruction.

      // The following line is equivalent to:
      //   overflow_bytes_ = total_bytes_read_ + buffer_size - INT_MAX;
      // except that it avoids overflows.  Signed integer overflow has
      // undefined results according to the C standard.
      overflow_bytes_ = total_bytes_read_ - (INT_MAX - buffer_size);
      buffer_end_ -= overflow_bytes_;
      total_bytes_read_ = INT_MAX;
    }

    RecomputeBufferLimits();
    return true;
  } else {
    buffer_ = NULL;
    buffer_end_ = NULL;
    return false;
  }
}

// CodedOutputStream =================================================

google::protobuf::internal::AtomicWord CodedOutputStream::default_serialization_deterministic_ = 0;

CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output)
  : output_(output),
    buffer_(NULL),
    buffer_size_(0),
    total_bytes_(0),
    had_error_(false),
    aliasing_enabled_(false),
    serialization_deterministic_is_overridden_(false) {
  // Eagerly Refresh() so buffer space is immediately available.
  Refresh();
  // The Refresh() may have failed. If the client doesn't write any data,
  // though, don't consider this an error. If the client does write data, then
  // another Refresh() will be attempted and it will set the error once again.
  had_error_ = false;
}

CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output,
                                     bool do_eager_refresh)
  : output_(output),
    buffer_(NULL),
    buffer_size_(0),
    total_bytes_(0),
    had_error_(false),
    aliasing_enabled_(false),
    serialization_deterministic_is_overridden_(false) {
  if (do_eager_refresh) {
    // Eagerly Refresh() so buffer space is immediately available.
    Refresh();
    // The Refresh() may have failed. If the client doesn't write any data,
    // though, don't consider this an error. If the client does write data, then
    // another Refresh() will be attempted and it will set the error once again.
    had_error_ = false;
  }
}

CodedOutputStream::~CodedOutputStream() {
  Trim();
}

void CodedOutputStream::Trim() {
  if (buffer_size_ > 0) {
    output_->BackUp(buffer_size_);
    total_bytes_ -= buffer_size_;
    buffer_size_ = 0;
    buffer_ = NULL;
  }
}

bool CodedOutputStream::Skip(int count) {
  if (count < 0) return false;

  while (count > buffer_size_) {
    count -= buffer_size_;
    if (!Refresh()) return false;
  }

  Advance(count);
  return true;
}

bool CodedOutputStream::GetDirectBufferPointer(void** data, int* size) {
  if (buffer_size_ == 0 && !Refresh()) return false;

  *data = buffer_;
  *size = buffer_size_;
  return true;
}

void CodedOutputStream::WriteRaw(const void* data, int size) {
  while (buffer_size_ < size) {
    memcpy(buffer_, data, buffer_size_);
    size -= buffer_size_;
    data = reinterpret_cast<const uint8*>(data) + buffer_size_;
    if (!Refresh()) return;
  }

  memcpy(buffer_, data, size);
  Advance(size);
}

uint8* CodedOutputStream::WriteRawToArray(
    const void* data, int size, uint8* target) {
  memcpy(target, data, size);
  return target + size;
}

void CodedOutputStream::WriteAliasedRaw(const void* data, int size) {
  if (size < buffer_size_
      ) {
    WriteRaw(data, size);
  } else {
    Trim();

    total_bytes_ += size;
    had_error_ |= !output_->WriteAliasedRaw(data, size);
  }
}

void CodedOutputStream::WriteLittleEndian32(uint32 value) {
  uint8 bytes[sizeof(value)];

  bool use_fast = buffer_size_ >= sizeof(value);
  uint8* ptr = use_fast ? buffer_ : bytes;

  WriteLittleEndian32ToArray(value, ptr);

  if (use_fast) {
    Advance(sizeof(value));
  } else {
    WriteRaw(bytes, sizeof(value));
  }
}

void CodedOutputStream::WriteLittleEndian64(uint64 value) {
  uint8 bytes[sizeof(value)];

  bool use_fast = buffer_size_ >= sizeof(value);
  uint8* ptr = use_fast ? buffer_ : bytes;

  WriteLittleEndian64ToArray(value, ptr);

  if (use_fast) {
    Advance(sizeof(value));
  } else {
    WriteRaw(bytes, sizeof(value));
  }
}

void CodedOutputStream::WriteVarint32SlowPath(uint32 value) {
  uint8 bytes[kMaxVarint32Bytes];
  uint8* target = &bytes[0];
  uint8* end = WriteVarint32ToArray(value, target);
  int size = end - target;
  WriteRaw(bytes, size);
}

void CodedOutputStream::WriteVarint64SlowPath(uint64 value) {
  uint8 bytes[kMaxVarintBytes];
  uint8* target = &bytes[0];
  uint8* end = WriteVarint64ToArray(value, target);
  int size = end - target;
  WriteRaw(bytes, size);
}

bool CodedOutputStream::Refresh() {
  void* void_buffer;
  if (output_->Next(&void_buffer, &buffer_size_)) {
    buffer_ = reinterpret_cast<uint8*>(void_buffer);
    total_bytes_ += buffer_size_;
    return true;
  } else {
    buffer_ = NULL;
    buffer_size_ = 0;
    had_error_ = true;
    return false;
  }
}

uint8* CodedOutputStream::WriteStringWithSizeToArray(const string& str,
                                                     uint8* target) {
  GOOGLE_DCHECK_LE(str.size(), kuint32max);
  target = WriteVarint32ToArray(str.size(), target);
  return WriteStringToArray(str, target);
}

}  // namespace io
}  // namespace protobuf
}  // namespace google

mime_type: text/plain


text:
#!/bin/sh -x
#
# Protocol Buffers - Google's data interchange format
# Copyright 2009 Google Inc.  All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Author: brianolson@google.com (Brian Olson)
#
# Test compatibility between command line gzip/gunzip binaries and
# ZeroCopyStream versions.

TESTFILE=Makefile

(./zcgzip < ${TESTFILE} | gunzip | cmp - ${TESTFILE}) && \
(gzip < ${TESTFILE} | ./zcgunzip | cmp - ${TESTFILE})

# Result of "(cmd) && (cmd)" implicitly becomes result of this script
# and thus the test.


text:
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Author: kenton@google.com (Kenton Varda)
//  Based on original Protocol Buffers design by
//  Sanjay Ghemawat, Jeff Dean, and others.
//
// Utility class for writing text to a ZeroCopyOutputStream.

#ifndef GOOGLE_PROTOBUF_IO_PRINTER_H__
#define GOOGLE_PROTOBUF_IO_PRINTER_H__

#include <string>
#include <map>
#include <vector>
#include <google/protobuf/stubs/common.h>

namespace google {
namespace protobuf {
namespace io {

class ZeroCopyOutputStream;     // zero_copy_stream.h

// Records annotations about a Printer's output.
class LIBPROTOBUF_EXPORT AnnotationCollector {
 public:
  // Records that the bytes in file_path beginning with begin_offset and ending
  // before end_offset are associated with the SourceCodeInfo-style path.
  virtual void AddAnnotation(size_t begin_offset, size_t end_offset,
                             const string& file_path,
                             const std::vector<int>& path) = 0;

  virtual ~AnnotationCollector() {}
};

// Records annotations about a Printer's output to the given protocol buffer,
// assuming that the buffer has an ::Annotation message exposing path,
// source_file, begin and end fields.
template <typename AnnotationProto>
class AnnotationProtoCollector : public AnnotationCollector {
 public:
  // annotation_proto is the protocol buffer to which new Annotations should be
  // added. It is not owned by the AnnotationProtoCollector.
  explicit AnnotationProtoCollector(AnnotationProto* annotation_proto)
      : annotation_proto_(annotation_proto) {}

  // Override for AnnotationCollector::AddAnnotation.
  virtual void AddAnnotation(size_t begin_offset, size_t end_offset,
                             const string& file_path,
                             const std::vector<int>& path) {
    typename AnnotationProto::Annotation* annotation =
        annotation_proto_->add_annotation();
    for (int i = 0; i < path.size(); ++i) {
      annotation->add_path(path[i]);
    }
    annotation->set_source_file(file_path);
    annotation->set_begin(begin_offset);
    annotation->set_end(end_offset);
  }

 private:
  // The protocol buffer to which new annotations should be added.
  AnnotationProto* const annotation_proto_;
};

// This simple utility class assists in code generation.  It basically
// allows the caller to define a set of variables and then output some
// text with variable substitutions.  Example usage:
//
//   Printer printer(output, '$');
//   map<string, string> vars;
//   vars["name"] = "Bob";
//   printer.Print(vars, "My name is $name$.");
//
// The above writes "My name is Bob." to the output stream.
//
// Printer aggressively enforces correct usage, crashing (with assert failures)
// in the case of undefined variables in debug builds. This helps greatly in
// debugging code which uses it.
//
// If a Printer is constructed with an AnnotationCollector, it will provide it
// with annotations that connect the Printer's output to paths that can identify
// various descriptors.  In the above example, if person_ is a descriptor that
// identifies Bob, we can associate the output string "My name is Bob." with
// a source path pointing to that descriptor with:
//
//   printer.Annotate("name", person_);
//
// The AnnotationCollector will be sent an annotation linking the output range
// covering "Bob" to the logical path provided by person_.  Tools may use
// this association to (for example) link "Bob" in the output back to the
// source file that defined the person_ descriptor identifying Bob.
//
// Annotate can only examine variables substituted during the last call to
// Print.  It is invalid to refer to a variable that was used multiple times
// in a single Print call.
//
// In full generality, one may specify a range of output text using a beginning
// substitution variable and an ending variable.  The resulting annotation will
// span from the first character of the substituted value for the beginning
// variable to the last character of the substituted value for the ending
// variable.  For example, the Annotate call above is equivalent to this one:
//
//   printer.Annotate("name", "name", person_);
//
// This is useful if multiple variables combine to form a single span of output
// that should be annotated with the same source path.  For example:
//
//   Printer printer(output, '$');
//   map<string, string> vars;
//   vars["first"] = "Alice";
//   vars["last"] = "Smith";
//   printer.Print(vars, "My name is $first$ $last$.");
//   printer.Annotate("first", "last", person_);
//
// This code would associate the span covering "Alice Smith" in the output with
// the person_ descriptor.
//
// Note that the beginning variable must come before (or overlap with, in the
// case of zero-sized substitution values) the ending variable.
//
// It is also sometimes useful to use variables with zero-sized values as
// markers.  This avoids issues with multiple references to the same variable
// and also allows annotation ranges to span literal text from the Print
// templates:
//
//   Printer printer(output, '$');
//   map<string, string> vars;
//   vars["foo"] = "bar";
//   vars["function"] = "call";
//   vars["mark"] = "";
//   printer.Print(vars, "$function$($foo$,$foo$)$mark$");
//   printer.Annotate("function", "mark", call_);
//
// This code associates the span covering "call(bar,bar)" in the output with the
// call_ descriptor.

class LIBPROTOBUF_EXPORT Printer {
 public:
  // Create a printer that writes text to the given output stream.  Use the
  // given character as the delimiter for variables.
  Printer(ZeroCopyOutputStream* output, char variable_delimiter);

  // Create a printer that writes text to the given output stream.  Use the
  // given character as the delimiter for variables.  If annotation_collector
  // is not null, Printer will provide it with annotations about code written
  // to the stream.  annotation_collector is not owned by Printer.
  Printer(ZeroCopyOutputStream* output, char variable_delimiter,
          AnnotationCollector* annotation_collector);

  ~Printer();

  // Link a subsitution variable emitted by the last call to Print to the object
  // described by descriptor.
  template <typename SomeDescriptor>
  void Annotate(const char* varname, const SomeDescriptor* descriptor) {
    Annotate(varname, varname, descriptor);
  }

  // Link the output range defined by the substitution variables as emitted by
  // the last call to Print to the object described by descriptor. The range
  // begins at begin_varname's value and ends after the last character of the
  // value substituted for end_varname.
  template <typename SomeDescriptor>
  void Annotate(const char* begin_varname, const char* end_varname,
                const SomeDescriptor* descriptor) {
    if (annotation_collector_ == NULL) {
      // Annotations aren't turned on for this Printer, so don't pay the cost
      // of building the location path.
      return;
    }
    std::vector<int> path;
    descriptor->GetLocationPath(&path);
    Annotate(begin_varname, end_varname, descriptor->file()->name(), path);
  }

  // Link a subsitution variable emitted by the last call to Print to the file
  // with path file_name.
  void Annotate(const char* varname, const string& file_name) {
    Annotate(varname, varname, file_name);
  }

  // Link the output range defined by the substitution variables as emitted by
  // the last call to Print to the file with path file_name. The range begins
  // at begin_varname's value and ends after the last character of the value
  // substituted for end_varname.
  void Annotate(const char* begin_varname, const char* end_varname,
                const string& file_name) {
    if (annotation_collector_ == NULL) {
      // Annotations aren't turned on for this Printer.
      return;
    }
    std::vector<int> empty_path;
    Annotate(begin_varname, end_varname, file_name, empty_path);
  }

  // Print some text after applying variable substitutions.  If a particular
  // variable in the text is not defined, this will crash.  Variables to be
  // substituted are identified by their names surrounded by delimiter
  // characters (as given to the constructor).  The variable bindings are
  // defined by the given map.
  void Print(const std::map<string, string>& variables, const char* text);

  // Like the first Print(), except the substitutions are given as parameters.
  void Print(const char* text);
  // Like the first Print(), except the substitutions are given as parameters.
  void Print(const char* text, const char* variable, const string& value);
  // Like the first Print(), except the substitutions are given as parameters.
  void Print(const char* text, const char* variable1, const string& value1,
                               const char* variable2, const string& value2);
  // Like the first Print(), except the substitutions are given as parameters.
  void Print(const char* text, const char* variable1, const string& value1,
                               const char* variable2, const string& value2,
                               const char* variable3, const string& value3);
  // Like the first Print(), except the substitutions are given as parameters.
  void Print(const char* text, const char* variable1, const string& value1,
                               const char* variable2, const string& value2,
                               const char* variable3, const string& value3,
                               const char* variable4, const string& value4);
  // Like the first Print(), except the substitutions are given as parameters.
  void Print(const char* text, const char* variable1, const string& value1,
                               const char* variable2, const string& value2,
                               const char* variable3, const string& value3,
                               const char* variable4, const string& value4,
                               const char* variable5, const string& value5);
  // Like the first Print(), except the substitutions are given as parameters.
  void Print(const char* text, const char* variable1, const string& value1,
                               const char* variable2, const string& value2,
                               const char* variable3, const string& value3,
                               const char* variable4, const string& value4,
                               const char* variable5, const string& value5,
                               const char* variable6, const string& value6);
  // Like the first Print(), except the substitutions are given as parameters.
  void Print(const char* text, const char* variable1, const string& value1,
                               const char* variable2, const string& value2,
                               const char* variable3, const string& value3,
                               const char* variable4, const string& value4,
                               const char* variable5, const string& value5,
                               const char* variable6, const string& value6,
                               const char* variable7, const string& value7);
  // Like the first Print(), except the substitutions are given as parameters.
  void Print(const char* text, const char* variable1, const string& value1,
                               const char* variable2, const string& value2,
                               const char* variable3, const string& value3,
                               const char* variable4, const string& value4,
                               const char* variable5, const string& value5,
                               const char* variable6, const string& value6,
                               const char* variable7, const string& value7,
                               const char* variable8, const string& value8);

  // Indent text by two spaces.  After calling Indent(), two spaces will be
  // inserted at the beginning of each line of text.  Indent() may be called
  // multiple times to produce deeper indents.
  void Indent();

  // Reduces the current indent level by two spaces, or crashes if the indent
  // level is zero.
  void Outdent();

  // Write a string to the output buffer.
  // This method does not look for newlines to add indentation.
  void PrintRaw(const string& data);

  // Write a zero-delimited string to output buffer.
  // This method does not look for newlines to add indentation.
  void PrintRaw(const char* data);

  // Write some bytes to the output buffer.
  // This method does not look for newlines to add indentation.
  void WriteRaw(const char* data, int size);

  // True if any write to the underlying stream failed.  (We don't just
  // crash in this case because this is an I/O failure, not a programming
  // error.)
  bool failed() const { return failed_; }

 private:
  // Link the output range defined by the substitution variables as emitted by
  // the last call to Print to the object found at the SourceCodeInfo-style path
  // in a file with path file_path. The range begins at the start of
  // begin_varname's value and ends after the last character of the value
  // substituted for end_varname. Note that begin_varname and end_varname
  // may refer to the same variable.
  void Annotate(const char* begin_varname, const char* end_varname,
                const string& file_path, const std::vector<int>& path);

  // Copy size worth of bytes from data to buffer_.
  void CopyToBuffer(const char* data, int size);

  const char variable_delimiter_;

  ZeroCopyOutputStream* const output_;
  char* buffer_;
  int buffer_size_;
  // The current position, in bytes, in the output stream.  This is equivalent
  // to the total number of bytes that have been written so far.  This value is
  // used to calculate annotation ranges in the substitutions_ map below.
  size_t offset_;

  string indent_;
  bool at_start_of_line_;
  bool failed_;

  // A map from variable name to [start, end) offsets in the output buffer.
  // These refer to the offsets used for a variable after the last call to
  // Print.  If a variable was used more than once, the entry used in
  // this map is set to a negative-length span.  For singly-used variables, the
  // start offset is the beginning of the substitution; the end offset is the
  // last byte of the substitution plus one (such that (end - start) is the
  // length of the substituted string).
  std::map<string, std::pair<size_t, size_t> > substitutions_;

  // Keeps track of the keys in substitutions_ that need to be updated when
  // indents are inserted. These are keys that refer to the beginning of the
  // current line.
  std::vector<string> line_start_variables_;

  // Returns true and sets range to the substitution range in the output for
  // varname if varname was used once in the last call to Print. If varname
  // was not used, or if it was used multiple times, returns false (and
  // fails a debug assertion).
  bool GetSubstitutionRange(const char* varname,
                            std::pair<size_t, size_t>* range);

  // If non-null, annotation_collector_ is used to store annotations about
  // generated code.
  AnnotationCollector* const annotation_collector_;

  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Printer);
};

}  // namespace io
}  // namespace protobuf

}  // namespace google
#endif  // GOOGLE_PROTOBUF_IO_PRINTER_H__


text:
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Author: brianolson@google.com (Brian Olson)
//
// This file contains the definition for classes GzipInputStream and
// GzipOutputStream.
//
// GzipInputStream decompresses data from an underlying
// ZeroCopyInputStream and provides the decompressed data as a
// ZeroCopyInputStream.
//
// GzipOutputStream is an ZeroCopyOutputStream that compresses data to
// an underlying ZeroCopyOutputStream.

#ifndef GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__
#define GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__

#include <google/protobuf/stubs/common.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <zlib.h>

namespace google {
namespace protobuf {
namespace io {

// A ZeroCopyInputStream that reads compressed data through zlib
class LIBPROTOBUF_EXPORT GzipInputStream : public ZeroCopyInputStream {
 public:
  // Format key for constructor
  enum Format {
    // zlib will autodetect gzip header or deflate stream
    AUTO = 0,

    // GZIP streams have some extra header data for file attributes.
    GZIP = 1,

    // Simpler zlib stream format.
    ZLIB = 2,
  };

  // buffer_size and format may be -1 for default of 64kB and GZIP format
  explicit GzipInputStream(
      ZeroCopyInputStream* sub_stream,
      Format format = AUTO,
      int buffer_size = -1);
  virtual ~GzipInputStream();

  // Return last error message or NULL if no error.
  inline const char* ZlibErrorMessage() const {
    return zcontext_.msg;
  }
  inline int ZlibErrorCode() const {
    return zerror_;
  }

  // implements ZeroCopyInputStream ----------------------------------
  bool Next(const void** data, int* size);
  void BackUp(int count);
  bool Skip(int count);
  int64 ByteCount() const;

 private:
  Format format_;

  ZeroCopyInputStream* sub_stream_;

  z_stream zcontext_;
  int zerror_;

  void* output_buffer_;
  void* output_position_;
  size_t output_buffer_length_;
  int64 byte_count_;

  int Inflate(int flush);
  void DoNextOutput(const void** data, int* size);

  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(GzipInputStream);
};

class LIBPROTOBUF_EXPORT GzipOutputStream : public ZeroCopyOutputStream {
 public:
  // Format key for constructor
  enum Format {
    // GZIP streams have some extra header data for file attributes.
    GZIP = 1,

    // Simpler zlib stream format.
    ZLIB = 2,
  };

  struct Options {
    // Defaults to GZIP.
    Format format;

    // What size buffer to use internally.  Defaults to 64kB.
    int buffer_size;

    // A number between 0 and 9, where 0 is no compression and 9 is best
    // compression.  Defaults to Z_DEFAULT_COMPRESSION (see zlib.h).
    int compression_level;

    // Defaults to Z_DEFAULT_STRATEGY.  Can also be set to Z_FILTERED,
    // Z_HUFFMAN_ONLY, or Z_RLE.  See the documentation for deflateInit2 in
    // zlib.h for definitions of these constants.
    int compression_strategy;

    Options();  // Initializes with default values.
  };

  // Create a GzipOutputStream with default options.
  explicit GzipOutputStream(ZeroCopyOutputStream* sub_stream);

  // Create a GzipOutputStream with the given options.
  GzipOutputStream(
      ZeroCopyOutputStream* sub_stream,
      const Options& options);

  virtual ~GzipOutputStream();

  // Return last error message or NULL if no error.
  inline const char* ZlibErrorMessage() const {
    return zcontext_.msg;
  }
  inline int ZlibErrorCode() const {
    return zerror_;
  }

  // Flushes data written so far to zipped data in the underlying stream.
  // It is the caller's responsibility to flush the underlying stream if
  // necessary.
  // Compression may be less efficient stopping and starting around flushes.
  // Returns true if no error.
  //
  // Please ensure that block size is > 6. Here is an excerpt from the zlib
  // doc that explains why:
  //
  // In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that avail_out
  // is greater than six to avoid repeated flush markers due to
  // avail_out == 0 on return.
  bool Flush();

  // Writes out all data and closes the gzip stream.
  // It is the caller's responsibility to close the underlying stream if
  // necessary.
  // Returns true if no error.
  bool Close();

  // implements ZeroCopyOutputStream ---------------------------------
  bool Next(void** data, int* size);
  void BackUp(int count);
  int64 ByteCount() const;

 private:
  ZeroCopyOutputStream* sub_stream_;
  // Result from calling Next() on sub_stream_
  void* sub_data_;
  int sub_data_size_;

  z_stream zcontext_;
  int zerror_;
  void* input_buffer_;
  size_t input_buffer_length_;

  // Shared constructor code.
  void Init(ZeroCopyOutputStream* sub_stream, const Options& options);

  // Do some compression.
  // Takes zlib flush mode.
  // Returns zlib error code.
  int Deflate(int flush);

  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(GzipOutputStream);
};

}  // namespace io
}  // namespace protobuf

}  // namespace google
#endif  // GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__


text:
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Author: kenton@google.com (Kenton Varda)
//  Based on original Protocol Buffers design by
//  Sanjay Ghemawat, Jeff Dean, and others.
//
// This file exists solely to document the google::protobuf::io namespace.
// It is not compiled into anything, but it may be read by an automated
// documentation generator.

namespace google {

namespace protobuf {

// Auxiliary classes used for I/O.
//
// The Protocol Buffer library uses the classes in this package to deal with
// I/O and encoding/decoding raw bytes.  Most users will not need to
// deal with this package.  However, users who want to adapt the system to
// work with their own I/O abstractions -- e.g., to allow Protocol Buffers
// to be read from a different kind of input stream without the need for a
// temporary buffer -- should take a closer look.
namespace io {}

}  // namespace protobuf
}  // namespace google