[prev in list] [next in list] [prev in thread] [next in thread]
List: avro-commits
Subject: svn commit: r935366 - in /hadoop/avro/trunk: ./ lang/c++/ lang/c++/api/ lang/c++/impl/ lang/c++/scri
From: sbanacho () apache ! org
Date: 2010-04-18 17:18:18
Message-ID: 20100418171818.772F123888CD () eris ! apache ! org
[Download RAW message or body]
Author: sbanacho
Date: Sun Apr 18 17:18:17 2010
New Revision: 935366
URL: http://svn.apache.org/viewvc?rev=935366&view=rev
Log:
AVRO-520. Refactor C++ validation code.
Removed:
hadoop/avro/trunk/lang/c++/api/ValidatingReader.hh
hadoop/avro/trunk/lang/c++/api/ValidatingWriter.hh
hadoop/avro/trunk/lang/c++/impl/ValidatingReader.cc
hadoop/avro/trunk/lang/c++/impl/ValidatingWriter.cc
Modified:
hadoop/avro/trunk/CHANGES.txt
hadoop/avro/trunk/lang/c++/MainPage.dox
hadoop/avro/trunk/lang/c++/Makefile.am
hadoop/avro/trunk/lang/c++/api/Parser.hh
hadoop/avro/trunk/lang/c++/api/Reader.hh
hadoop/avro/trunk/lang/c++/api/Resolver.hh
hadoop/avro/trunk/lang/c++/api/ResolverSchema.hh
hadoop/avro/trunk/lang/c++/api/Serializer.hh
hadoop/avro/trunk/lang/c++/api/Types.hh
hadoop/avro/trunk/lang/c++/api/Validator.hh
hadoop/avro/trunk/lang/c++/api/Writer.hh
hadoop/avro/trunk/lang/c++/impl/Validator.cc
hadoop/avro/trunk/lang/c++/scripts/gen-cppcode.py
hadoop/avro/trunk/lang/c++/test/testgen.cc
hadoop/avro/trunk/lang/c++/test/unittest.cc
Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Sun Apr 18 17:18:17 2010
@@ -25,6 +25,7 @@ Avro 1.4.0 (unreleased)
AVRO-514. Removing unnecessary ruby StringIO calls. (jmhodges)
+ AVRO-520. Refactor C++ validation code. (sbanacho)
BUG FIXES
AVRO-461. Skipping primitives in the ruby side (jmhodges)
Modified: hadoop/avro/trunk/lang/c++/MainPage.dox
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/MainPage.dox?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/MainPage.dox (original)
+++ hadoop/avro/trunk/lang/c++/MainPage.dox Sun Apr 18 17:18:17 2010
@@ -139,6 +139,7 @@ inline void serialize(Serializer &s,
s.writeRecord();
serialize(s, val.real);
serialize(s, val.imaginary);
+ s.writeRecordEnd();
}
template <typename Parser>
@@ -146,6 +147,7 @@ inline void parse(Parser &p, complex
p.readRecord();
parse(p, val.real);
parse(p, val.imaginary);
+ p.readRecordEnd();
}</PRE><P>
It also adds the following to the avro namespace:</P>
<PRE>template <> struct is_serializable<Math::complex> : public \
boost::true_type{};</PRE><P> @@ -222,10 +224,10 @@ the API:</P>
// Manually parse data, the Parser object binds the data to the schema
avro::Parser<ValidatingReader> parser(mySchema, myData);
- assert( parser.nextType() == AVRO_READER);
+ assert( parser.nextType() == AVRO_RECORD);
// Begin parsing
- parser.beginRecord();
+ parser.readRecord();
Math::complex c;
@@ -242,6 +244,8 @@ the API:</P>
std::cout << â I did not expect that!\nâ
}
}
+
+ parser.readRecordEnd();
}</PRE><P>
The above code shows that if you don't know the schema at compile
time, you can still write code that parses the data, by reading the
Modified: hadoop/avro/trunk/lang/c++/Makefile.am
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/Makefile.am?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/Makefile.am (original)
+++ hadoop/avro/trunk/lang/c++/Makefile.am Sun Apr 18 17:18:17 2010
@@ -33,8 +33,6 @@ api/Serializer.hh \
api/SymbolMap.hh \
api/Types.hh \
api/ValidSchema.hh \
-api/ValidatingReader.hh \
-api/ValidatingWriter.hh \
api/Validator.hh \
api/Writer.hh \
api/Zigzag.hh \
@@ -74,8 +72,6 @@ impl/ResolverSchema.cc \
impl/Schema.cc \
impl/Types.cc \
impl/ValidSchema.cc \
-impl/ValidatingReader.cc \
-impl/ValidatingWriter.cc \
impl/Validator.cc \
impl/Zigzag.cc \
parser/AvroYacc.yy \
@@ -107,10 +103,10 @@ buffertest_LDADD = $(top_builddir)/libav
dist-hook:
find $(distdir) -name '.svn' | xargs rm -rf
-testgen.hh : bigrecord.precompile
+testgen.hh : bigrecord.precompile $(top_srcdir)/scripts/gen-cppcode.py
$(PYTHON) $(top_srcdir)/scripts/gen-cppcode.py -n testgen -i $< -o $@
-testgen2.hh : bigrecord2.precompile
+testgen2.hh : bigrecord2.precompile $(top_srcdir)/scripts/gen-cppcode.py
$(PYTHON) $(top_srcdir)/scripts/gen-cppcode.py -n testgen2 -i $< -o $@
bigrecord.precompile: $(top_srcdir)/jsonschemas/bigrecord precompile$(EXEEXT)
Modified: hadoop/avro/trunk/lang/c++/api/Parser.hh
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/api/Parser.hh?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/api/Parser.hh (original)
+++ hadoop/avro/trunk/lang/c++/api/Parser.hh Sun Apr 18 17:18:17 2010
@@ -20,7 +20,6 @@
#define avro_Parser_hh__
#include "Reader.hh"
-#include "ValidatingReader.hh"
namespace avro {
@@ -102,6 +101,10 @@ class Parser : private boost::noncopyabl
reader_.readRecord();
}
+ void readRecordEnd() {
+ reader_.readRecordEnd();
+ }
+
int64_t readArrayBlockSize() {
return reader_.readArrayBlockSize();
}
Modified: hadoop/avro/trunk/lang/c++/api/Reader.hh
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/api/Reader.hh?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/api/Reader.hh (original)
+++ hadoop/avro/trunk/lang/c++/api/Reader.hh Sun Apr 18 17:18:17 2010
@@ -25,6 +25,7 @@
#include "Zigzag.hh"
#include "Types.hh"
+#include "Validator.hh"
#include "buffer/BufferReader.hh"
namespace avro {
@@ -34,34 +35,46 @@ namespace avro {
/// in the avro binary data is the expected type.
///
-class Reader : private boost::noncopyable
+template<class ValidatorType>
+class ReaderImpl : private boost::noncopyable
{
public:
- explicit Reader(const InputBuffer &buffer) :
+ explicit ReaderImpl(const InputBuffer &buffer) :
reader_(buffer)
{}
- void readValue(Null &) {}
+ ReaderImpl(const ValidSchema &schema, const InputBuffer &buffer) :
+ validator_(schema),
+ reader_(buffer)
+ {}
+
+ void readValue(Null &) {
+ validator_.checkTypeExpected(AVRO_NULL);
+ }
void readValue(bool &val) {
+ validator_.checkTypeExpected(AVRO_BOOL);
uint8_t ival;
reader_.read(ival);
val = (ival != 0);
}
void readValue(int32_t &val) {
+ validator_.checkTypeExpected(AVRO_INT);
uint32_t encoded = readVarInt();
val = decodeZigzag32(encoded);
}
void readValue(int64_t &val) {
+ validator_.checkTypeExpected(AVRO_LONG);
uint64_t encoded = readVarInt();
val = decodeZigzag64(encoded);
}
void readValue(float &val) {
+ validator_.checkTypeExpected(AVRO_FLOAT);
union {
float f;
uint32_t i;
@@ -71,6 +84,7 @@ class Reader : private boost::noncopyabl
}
void readValue(double &val) {
+ validator_.checkTypeExpected(AVRO_DOUBLE);
union {
double d;
uint64_t i;
@@ -80,11 +94,13 @@ class Reader : private boost::noncopyabl
}
void readValue(std::string &val) {
+ validator_.checkTypeExpected(AVRO_STRING);
int64_t size = readSize();
reader_.read(val, size);
}
void readBytes(std::vector<uint8_t> &val) {
+ validator_.checkTypeExpected(AVRO_BYTES);
int64_t size = readSize();
val.reserve(size);
@@ -96,45 +112,66 @@ class Reader : private boost::noncopyabl
}
void readFixed(uint8_t *val, size_t size) {
+ validator_.checkFixedSizeExpected(size);
reader_.read(reinterpret_cast<char *>(val), size);
}
template <size_t N>
void readFixed(uint8_t (&val)[N]) {
- readFixed(val, N);
+ this->readFixed(val, N);
}
template <size_t N>
void readFixed(boost::array<uint8_t, N> &val) {
- readFixed(val.c_array(), N);
+ this->readFixed(val.c_array(), N);
}
- void readRecord() { }
+ void readRecord() {
+ validator_.checkTypeExpected(AVRO_RECORD);
+ validator_.checkTypeExpected(AVRO_LONG);
+ validator_.setCount(1);
+ }
+
+ void readRecordEnd() {
+ validator_.checkTypeExpected(AVRO_RECORD);
+ validator_.checkTypeExpected(AVRO_LONG);
+ validator_.setCount(0);
+ }
int64_t readArrayBlockSize() {
- return readSize();
+ validator_.checkTypeExpected(AVRO_ARRAY);
+ return readCount();
}
int64_t readUnion() {
- return readSize();
+ validator_.checkTypeExpected(AVRO_UNION);
+ return readCount();
}
int64_t readEnum() {
- return readSize();
+ validator_.checkTypeExpected(AVRO_ENUM);
+ return readCount();
}
int64_t readMapBlockSize() {
- return readSize();
+ validator_.checkTypeExpected(AVRO_MAP);
+ return readCount();
}
- private:
+ Type nextType() const {
+ return validator_.nextTypeExpected();
+ }
- int64_t readSize() {
- int64_t size(0);
- readValue(size);
- return size;
+ bool currentRecordName(std::string &name) const {
+ return validator_.getCurrentRecordName(name);
}
+ bool nextFieldName(std::string &name) const {
+ return validator_.getNextFieldName(name);
+ }
+
+ private:
+
uint64_t readVarInt() {
uint64_t encoded = 0;
uint8_t val = 0;
@@ -149,10 +186,26 @@ class Reader : private boost::noncopyabl
return encoded;
}
- BufferReader reader_;
+ int64_t readSize() {
+ uint64_t encoded = readVarInt();
+ int64_t size = decodeZigzag64(encoded);
+ return size;
+ }
+
+ int64_t readCount() {
+ validator_.checkTypeExpected(AVRO_LONG);
+ int64_t count = readSize();
+ validator_.setCount(count);
+ return count;
+ }
+
+ ValidatorType validator_;
+ BufferReader reader_;
};
+typedef ReaderImpl<NullValidator> Reader;
+typedef ReaderImpl<Validator> ValidatingReader;
} // namespace avro
Modified: hadoop/avro/trunk/lang/c++/api/Resolver.hh
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/api/Resolver.hh?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/api/Resolver.hh (original)
+++ hadoop/avro/trunk/lang/c++/api/Resolver.hh Sun Apr 18 17:18:17 2010
@@ -23,12 +23,13 @@
#include <boost/noncopyable.hpp>
#include <stdint.h>
+#include "Reader.hh"
+
/// \file Resolver.hh
///
namespace avro {
-class Reader;
class ValidSchema;
class Layout;
Modified: hadoop/avro/trunk/lang/c++/api/ResolverSchema.hh
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/api/ResolverSchema.hh?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/api/ResolverSchema.hh (original)
+++ hadoop/avro/trunk/lang/c++/api/ResolverSchema.hh Sun Apr 18 17:18:17 2010
@@ -24,6 +24,7 @@
#include <boost/shared_ptr.hpp>
#include <stdint.h>
#include "Boost.hh"
+#include "Reader.hh"
/// \file ResolverSchema.hh
///
@@ -31,7 +32,6 @@
namespace avro {
class ValidSchema;
-class Reader;
class Layout;
class Resolver;
Modified: hadoop/avro/trunk/lang/c++/api/Serializer.hh
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/api/Serializer.hh?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/api/Serializer.hh (original)
+++ hadoop/avro/trunk/lang/c++/api/Serializer.hh Sun Apr 18 17:18:17 2010
@@ -22,7 +22,6 @@
#include <boost/noncopyable.hpp>
#include "Writer.hh"
-#include "ValidatingWriter.hh"
namespace avro {
@@ -91,6 +90,10 @@ class Serializer : private boost::noncop
writer_.writeRecord();
}
+ void writeRecordEnd() {
+ writer_.writeRecordEnd();
+ }
+
void writeArrayBlock(int64_t size) {
writer_.writeArrayBlock(size);
}
Modified: hadoop/avro/trunk/lang/c++/api/Types.hh
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/api/Types.hh?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/api/Types.hh (original)
+++ hadoop/avro/trunk/lang/c++/api/Types.hh Sun Apr 18 17:18:17 2010
@@ -45,7 +45,8 @@ enum Type {
// The following is a pseudo-type used in implementation
- AVRO_SYMBOLIC = AVRO_NUM_TYPES
+ AVRO_SYMBOLIC = AVRO_NUM_TYPES,
+ AVRO_UNKNOWN = -1
};
Modified: hadoop/avro/trunk/lang/c++/api/Validator.hh
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/api/Validator.hh?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/api/Validator.hh (original)
+++ hadoop/avro/trunk/lang/c++/api/Validator.hh Sun Apr 18 17:18:17 2010
@@ -28,6 +28,40 @@
namespace avro {
+class NullValidator : private boost::noncopyable
+{
+ public:
+
+ explicit NullValidator(const ValidSchema &schema) {}
+ NullValidator() {}
+
+ void setCount(int64_t val) {}
+
+ bool typeIsExpected(Type type) const {
+ return true;
+ }
+
+ Type nextTypeExpected() const {
+ return AVRO_UNKNOWN;
+ }
+
+ int nextSizeExpected() const {
+ return 0;
+ }
+
+ bool getCurrentRecordName(std::string &name) const {
+ return true;
+ }
+
+ bool getNextFieldName(std::string &name) const {
+ return true;
+ }
+
+ void checkTypeExpected(Type type) { }
+ void checkFixedSizeExpected(int size) { }
+
+
+};
/// This class is used by both the ValidatingSerializer and ValidationParser
/// objects. It advances the parse tree (containing logic how to advance
@@ -37,14 +71,11 @@ namespace avro {
class Validator : private boost::noncopyable
{
- typedef uint32_t flag_t;
-
public:
explicit Validator(const ValidSchema &schema);
- void advance();
- void advanceWithCount(int64_t val);
+ void setCount(int64_t val);
bool typeIsExpected(Type type) const {
return (expectedTypesFlag_ & typeToFlag(type));
@@ -59,8 +90,30 @@ class Validator : private boost::noncopy
bool getCurrentRecordName(std::string &name) const;
bool getNextFieldName(std::string &name) const;
+ void checkTypeExpected(Type type) {
+ if(! typeIsExpected(type)) {
+ throw Exception(
+ boost::format("Type %1% does not match schema %2%")
+ % type % nextType_
+ );
+ }
+ advance();
+ }
+
+ void checkFixedSizeExpected(int size) {
+ if( nextSizeExpected() != size) {
+ throw Exception(
+ boost::format("Wrong size for fixed, got %1%, expected %2%")
+ % size % nextSizeExpected()
+ );
+ }
+ checkTypeExpected(AVRO_FIXED);
+ }
+
private:
+ typedef uint32_t flag_t;
+
flag_t typeToFlag(Type type) const {
flag_t flag = (1L << type);
return flag;
@@ -70,8 +123,11 @@ class Validator : private boost::noncopy
void setWaitingForCount();
- void recordAdvance();
+ void advance();
+ void doAdvance();
+
void enumAdvance();
+ bool countingSetup();
void countingAdvance();
void unionAdvance();
void fixedAdvance();
@@ -91,7 +147,7 @@ class Validator : private boost::noncopy
node(n), pos(0)
{}
NodePtr node; ///< save the node
- size_t pos; ///< track the leaf position to visit
+ size_t pos; ///< track the leaf position to visit
};
std::vector<CompoundType> compoundStack_;
Modified: hadoop/avro/trunk/lang/c++/api/Writer.hh
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/api/Writer.hh?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/api/Writer.hh (original)
+++ hadoop/avro/trunk/lang/c++/api/Writer.hh Sun Apr 18 17:18:17 2010
@@ -24,38 +24,48 @@
#include "buffer/Buffer.hh"
#include "Zigzag.hh"
#include "Types.hh"
+#include "Validator.hh"
namespace avro {
/// Class for writing avro data to a stream.
-class Writer : private boost::noncopyable
+template<class ValidatorType>
+class WriterImpl : private boost::noncopyable
{
public:
- Writer() {}
+ WriterImpl() {}
- void writeValue(const Null &) {}
+ explicit WriterImpl(const ValidSchema &schema) :
+ validator_(schema)
+ {}
+
+ void writeValue(const Null &) {
+ validator_.checkTypeExpected(AVRO_NULL);
+ }
void writeValue(bool val) {
+ validator_.checkTypeExpected(AVRO_BOOL);
int8_t byte = (val != 0);
buffer_.writeTo(byte);
}
void writeValue(int32_t val) {
+ validator_.checkTypeExpected(AVRO_INT);
boost::array<uint8_t, 5> bytes;
size_t size = encodeInt32(val, bytes);
buffer_.writeTo(reinterpret_cast<const char *>(bytes.data()), size);
}
void writeValue(int64_t val) {
- boost::array<uint8_t, 10> bytes;
- size_t size = encodeInt64(val, bytes);
- buffer_.writeTo(reinterpret_cast<const char *>(bytes.data()), size);
+ validator_.checkTypeExpected(AVRO_LONG);
+ putLong(val);
}
void writeValue(float val) {
+ validator_.checkTypeExpected(AVRO_FLOAT);
union {
float f;
int32_t i;
@@ -66,6 +76,7 @@ class Writer : private boost::noncopyabl
}
void writeValue(double val) {
+ validator_.checkTypeExpected(AVRO_DOUBLE);
union {
double d;
int64_t i;
@@ -76,48 +87,65 @@ class Writer : private boost::noncopyabl
}
void writeValue(const std::string &val) {
- writeBytes(val.c_str(), val.size());
+ validator_.checkTypeExpected(AVRO_STRING);
+ putBytes(val.c_str(), val.size());
}
void writeBytes(const void *val, size_t size) {
- this->writeValue(static_cast<int64_t>(size));
- buffer_.writeTo(reinterpret_cast<const char *>(val), size);
+ validator_.checkTypeExpected(AVRO_BYTES);
+ putBytes(val, size);
}
template <size_t N>
void writeFixed(const uint8_t (&val)[N]) {
+ validator_.checkFixedSizeExpected(N);
buffer_.writeTo(reinterpret_cast<const char *>(val), N);
}
template <size_t N>
void writeFixed(const boost::array<uint8_t, N> &val) {
+ validator_.checkFixedSizeExpected(val.size());
buffer_.writeTo(reinterpret_cast<const char *>(val.data()), val.size());
}
- void writeRecord() {}
+ void writeRecord() {
+ validator_.checkTypeExpected(AVRO_RECORD);
+ validator_.checkTypeExpected(AVRO_LONG);
+ validator_.setCount(1);
+ }
+
+ void writeRecordEnd() {
+ validator_.checkTypeExpected(AVRO_RECORD);
+ validator_.checkTypeExpected(AVRO_LONG);
+ validator_.setCount(0);
+ }
void writeArrayBlock(int64_t size) {
- this->writeValue(static_cast<int64_t>(size));
+ validator_.checkTypeExpected(AVRO_ARRAY);
+ writeCount(size);
}
void writeArrayEnd() {
- buffer_.writeTo<uint8_t>(0);
+ writeArrayBlock(0);
}
void writeMapBlock(int64_t size) {
- this->writeValue(static_cast<int64_t>(size));
+ validator_.checkTypeExpected(AVRO_MAP);
+ writeCount(size);
}
void writeMapEnd() {
- buffer_.writeTo<uint8_t>(0);
+ writeMapBlock(0);
}
void writeUnion(int64_t choice) {
- this->writeValue(static_cast<int64_t>(choice));
+ validator_.checkTypeExpected(AVRO_UNION);
+ writeCount(choice);
}
void writeEnum(int64_t choice) {
- this->writeValue(static_cast<int64_t>(choice));
+ validator_.checkTypeExpected(AVRO_ENUM);
+ writeCount(choice);
}
InputBuffer buffer() const {
@@ -126,10 +154,31 @@ class Writer : private boost::noncopyabl
private:
+ void putLong(int64_t val) {
+ boost::array<uint8_t, 10> bytes;
+ size_t size = encodeInt64(val, bytes);
+ buffer_.writeTo(reinterpret_cast<const char *>(bytes.data()), size);
+ }
+
+ void putBytes(const void *val, size_t size) {
+ putLong(size);
+ buffer_.writeTo(reinterpret_cast<const char *>(val), size);
+ }
+
+ void writeCount(int64_t count) {
+ validator_.checkTypeExpected(AVRO_LONG);
+ validator_.setCount(count);
+ putLong(count);
+ }
+
+ ValidatorType validator_;
OutputBuffer buffer_;
};
+typedef WriterImpl<NullValidator> Writer;
+typedef WriterImpl<Validator> ValidatingWriter;
+
} // namespace avro
#endif
Modified: hadoop/avro/trunk/lang/c++/impl/Validator.cc
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/impl/Validator.cc?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/impl/Validator.cc (original)
+++ hadoop/avro/trunk/lang/c++/impl/Validator.cc Sun Apr 18 17:18:17 2010
@@ -45,26 +45,6 @@ Validator::setWaitingForCount()
}
void
-Validator::recordAdvance()
-{
- // record doesn't use this flag because it doesn't need to set
- // up anything at the start, but just clear it
- compoundStarted_ = false;
-
- // determine the next record entry to process
- size_t index = (compoundStack_.back().pos)++;
-
- const NodePtr &node = compoundStack_.back().node;
- if(index < node->leaves() ) {
- setupOperation(node->leafAt(index));
- }
- else {
- // done with this record, remove it from the processing stack
- compoundStack_.pop_back();
- }
-}
-
-void
Validator::enumAdvance()
{
if(compoundStarted_) {
@@ -77,28 +57,36 @@ Validator::enumAdvance()
}
}
-void
-Validator::countingAdvance()
+bool
+Validator::countingSetup()
{
- const NodePtr &node = compoundStack_.back().node;
-
+ bool proceed = true;
if(compoundStarted_) {
setWaitingForCount();
compoundStarted_ = false;
+ proceed = false;
}
else if(waitingForCount_) {
waitingForCount_ = false;
if(count_ == 0) {
compoundStack_.pop_back();
+ proceed = false;
}
else {
counters_.push_back(count_);
- setupOperation(node->leafAt(0));
}
}
- else {
- size_t index = ++(compoundStack_.back().pos);
+ return proceed;
+}
+
+void
+Validator::countingAdvance()
+{
+ if(countingSetup()) {
+
+ size_t index = (compoundStack_.back().pos)++;
+ const NodePtr &node = compoundStack_.back().node;
if(index < node->leaves() ) {
setupOperation(node->leafAt(index));
@@ -113,7 +101,8 @@ Validator::countingAdvance()
expectedTypesFlag_ = typeToFlag(nextType_);
}
else {
- setupOperation(node->leafAt(0));
+ size_t index = (compoundStack_.back().pos)++;
+ setupOperation(node->leafAt(index));
}
}
}
@@ -135,7 +124,11 @@ Validator::unionAdvance()
setupOperation(node->leafAt(count_));
}
else {
- throw Exception("Union out of range");
+ throw Exception(
+ boost::format("Union selection out of range, got %1%," \
+ " expecting 0-%2%")
+ % count_ % (node->leaves() -1)
+ );
}
}
}
@@ -154,7 +147,7 @@ Validator::nextSizeExpected() const
}
void
-Validator::advance()
+Validator::doAdvance()
{
typedef void (Validator::*AdvanceFunc)();
@@ -168,7 +161,7 @@ Validator::advance()
0, // double
0, // bool
0, // null
- &Validator::recordAdvance,
+ &Validator::countingAdvance, // Record is treated like counting with count \
== 1 &Validator::enumAdvance,
&Validator::countingAdvance,
&Validator::countingAdvance,
@@ -197,8 +190,15 @@ Validator::advance()
}
}
+void Validator::advance()
+{
+ if(!waitingForCount_) {
+ doAdvance();
+ }
+}
+
void
-Validator::advanceWithCount(int64_t count)
+Validator::setCount(int64_t count)
{
if(!waitingForCount_) {
throw Exception("Not expecting count");
@@ -208,7 +208,7 @@ Validator::advanceWithCount(int64_t coun
}
count_ = count;
- advance();
+ doAdvance();
}
void
Modified: hadoop/avro/trunk/lang/c++/scripts/gen-cppcode.py
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/scripts/gen-cppcode.py?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/scripts/gen-cppcode.py (original)
+++ hadoop/avro/trunk/lang/c++/scripts/gen-cppcode.py Sun Apr 18 17:18:17 2010
@@ -1,7 +1,6 @@
#!/usr/bin/python
-license = '''
-/**
+license = '''/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -88,13 +87,13 @@ $recordfields$};
template <typename Serializer>
inline void serialize(Serializer &s, const $name$ &val, const boost::true_type &) {
s.writeRecord();
-$serializefields$
+$serializefields$ s.writeRecordEnd();
}
template <typename Parser>
inline void parse(Parser &p, $name$ &val, const boost::true_type &) {
p.readRecord();
-$parsefields$
+$parsefields$ p.readRecordEnd();
}
class $name$_Layout : public avro::CompoundLayout {
@@ -181,8 +180,7 @@ template <typename Serializer>
inline void serialize(Serializer &s, const $name$ &val, const boost::true_type &) {
s.writeUnion(val.choice);
switch(val.choice) {
-$switchserialize$
- default :
+$switchserialize$ default :
throw avro::Exception("Unrecognized union choice");
}
}
@@ -191,8 +189,7 @@ template <typename Parser>
inline void parse(Parser &p, $name$ &val, const boost::true_type &) {
val.choice = p.readUnion();
switch(val.choice) {
-$switchparse$
- default :
+$switchparse$ default :
throw avro::Exception("Unrecognized union choice");
}
}
Modified: hadoop/avro/trunk/lang/c++/test/testgen.cc
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/test/testgen.cc?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/test/testgen.cc (original)
+++ hadoop/avro/trunk/lang/c++/test/testgen.cc Sun Apr 18 17:18:17 2010
@@ -27,9 +27,7 @@
#include "Serializer.hh"
#include "Writer.hh"
-#include "ValidatingWriter.hh"
#include "Reader.hh"
-#include "ValidatingReader.hh"
#include "Node.hh"
#include "ValidSchema.hh"
#include "Compiler.hh"
Modified: hadoop/avro/trunk/lang/c++/test/unittest.cc
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/c%2B%2B/test/unittest.cc?rev=935366&r1=935365&r2=935366&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/c++/test/unittest.cc (original)
+++ hadoop/avro/trunk/lang/c++/test/unittest.cc Sun Apr 18 17:18:17 2010
@@ -216,6 +216,7 @@ struct TestSchema
std::cout << "Record\n";
s.writeRecord();
s.writeFloat(-101.101f);
+ s.writeRecordEnd();
std::cout << "Bool\n";
s.writeBool(true);
@@ -229,6 +230,7 @@ struct TestSchema
std::cout << "Int\n";
s.writeInt(-3456);
+ s.writeRecordEnd();
}
void printEncoding() {
@@ -321,6 +323,7 @@ struct TestSchema
float f = p.readFloat();
std::cout << f << '\n';
BOOST_CHECK_EQUAL(f, -101.101f);
+ p.readRecordEnd();
}
template <typename Parser>
@@ -378,6 +381,7 @@ struct TestSchema
int32_t intval = p.readInt();
std::cout << intval << '\n';
BOOST_CHECK_EQUAL(intval, -3456);
+ p.readRecordEnd();
}
void readRawData() {
@@ -519,6 +523,7 @@ struct TestNested
s.writeUnion(0);
s.writeNull();
s.writeBool(true);
+ s.writeRecordEnd();
return s.buffer();
}
@@ -538,30 +543,43 @@ struct TestNested
s.writeRecord();
s.writeLong(3);
s.writeUnion(0);
+ {
+ s.writeNull();
+ }
+ s.writeBool(false);
+ s.writeRecordEnd();
}
- s.writeNull();
+ s.writeBool(false);
+ s.writeRecordEnd();
+
}
s.writeBool(true);
+ s.writeRecordEnd();
return s.buffer();
}
- void validatingParser(InputBuffer &buf)
+ void readRecord(Parser<ValidatingReader> &p)
{
- Parser<ValidatingReader> p(schema_, buf);
- int64_t val = 0;
- int64_t path = 0;
-
- do {
- p.readRecord();
- val = p.readLong();
- std::cout << "longval = " << val << '\n';
- path = p.readUnion();
- } while(path == 1);
-
- p.readNull();
+ p.readRecord();
+ int64_t val = p.readLong();
+ std::cout << "longval = " << val << '\n';
+ int64_t path = p.readUnion();
+ if (path == 1) {
+ readRecord(p);
+ }
+ else {
+ p.readNull();
+ }
bool b = p.readBool();
std::cout << "bval = " << b << '\n';
+ p.readRecordEnd();
+ }
+
+ void validatingParser(InputBuffer &buf)
+ {
+ Parser<ValidatingReader> p(schema_, buf);
+ readRecord(p);
}
void testToScreen() {
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic