[prev in list] [next in list] [prev in thread] [next in thread] 

List:       avro-commits
Subject:    svn commit: r1418584 - in /avro/trunk/lang/py: src/avro/datafile.py test/test_datafile.py
From:       tebeka () apache ! org
Date:       2012-12-08 1:05:16
Message-ID: 20121208010517.250EB23888E7 () eris ! apache ! org
[Download RAW message or body]

Author: tebeka
Date: Sat Dec  8 01:05:15 2012
New Revision: 1418584

URL: http://svn.apache.org/viewvc?rev=1418584&view=rev
Log:
AVRO-1217. Python: set_meta on DataFileWriter does not do anything

Modified:
    avro/trunk/lang/py/src/avro/datafile.py
    avro/trunk/lang/py/test/test_datafile.py

Modified: avro/trunk/lang/py/src/avro/datafile.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/datafile.py?rev=1418584&r1=1418583&r2=1418584&view=diff
 ==============================================================================
--- avro/trunk/lang/py/src/avro/datafile.py (original)
+++ avro/trunk/lang/py/src/avro/datafile.py Sat Dec  8 01:05:15 2012
@@ -86,6 +86,7 @@ class DataFileWriter(object):
     self._buffer_encoder = io.BinaryEncoder(self._buffer_writer)
     self._block_count = 0
     self._meta = {}
+    self._header_written = False
 
     if writers_schema is not None:
       if codec not in VALID_CODECS:
@@ -94,7 +95,6 @@ class DataFileWriter(object):
       self.set_meta('avro.codec', codec)
       self.set_meta('avro.schema', str(writers_schema))
       self.datum_writer.writers_schema = writers_schema
-      self._write_header()
     else:
       # open writer for reading to collect metadata
       dfr = DataFileReader(writer, io.DatumReader())
@@ -111,6 +111,7 @@ class DataFileWriter(object):
 
       # seek to the end of the file and prepare for writing
       writer.seek(0, 2)
+      self._header_written = True
 
   # read-only properties
   writer = property(lambda self: self._writer)
@@ -145,9 +146,13 @@ class DataFileWriter(object):
               'meta': self.meta,
               'sync': self.sync_marker}
     self.datum_writer.write_data(META_SCHEMA, header, self.encoder)
+    self._header_written = True
 
   # TODO(hammer): make a schema for blocks and use datum_writer
   def _write_block(self):
+    if not self._header_written:
+      self._write_header()
+
     if self.block_count > 0:
       # write number of items in block
       self.encoder.write_long(self.block_count)

Modified: avro/trunk/lang/py/test/test_datafile.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/test/test_datafile.py?rev=1418584&r1=1418583&r2=1418584&view=diff
 ==============================================================================
--- avro/trunk/lang/py/test/test_datafile.py (original)
+++ avro/trunk/lang/py/test/test_datafile.py Sat Dec  8 01:05:15 2012
@@ -51,7 +51,12 @@ SCHEMAS_TO_VALIDATE = (
 )
 
 FILENAME = 'test_datafile.out'
-CODECS_TO_VALIDATE = ('null', 'deflate', 'snappy')
+CODECS_TO_VALIDATE = ('null', 'deflate')
+try:
+  import snappy
+  CODECS_TO_VALIDATE += ('snappy',)
+except ImportError:
+  print 'Snappy not present, will skip testing it.'
 
 # TODO(hammer): clean up written files with ant, not os.remove
 class TestDataFile(unittest.TestCase):
@@ -63,13 +68,6 @@ class TestDataFile(unittest.TestCase):
     correct = 0
     for i, (example_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE):
       for codec in CODECS_TO_VALIDATE:
-        if (codec == 'snappy'):
-          try:
-            import snappy
-          except:
-            print 'Snappy not present. Skipping.'
-            correct += 1
-            continue
         print ''
         print 'SCHEMA NUMBER %d' % (i + 1)
         print '================'
@@ -112,13 +110,6 @@ class TestDataFile(unittest.TestCase):
     correct = 0
     for i, (example_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE):
       for codec in CODECS_TO_VALIDATE:
-        if (codec == 'snappy'):
-          try:
-            import snappy
-          except:
-            print 'Snappy not present. Skipping.'
-            correct += 1
-            continue
         print ''
         print 'SCHEMA NUMBER %d' % (i + 1)
         print '================'
@@ -184,5 +175,28 @@ class TestDataFile(unittest.TestCase):
         datums.append(datum)
     self.assertTrue(reader.closed)
 
+  def test_metadata(self):
+    # Test the writer with a 'with' statement.
+    writer = open(FILENAME, 'wb')
+    datum_writer = io.DatumWriter()
+    sample_schema, sample_datum = SCHEMAS_TO_VALIDATE[1]
+    schema_object = schema.parse(sample_schema)
+    with datafile.DataFileWriter(writer, datum_writer, schema_object) as dfw:
+      dfw.set_meta('test.string', 'foo')
+      dfw.set_meta('test.number', '1')
+      dfw.append(sample_datum)
+    self.assertTrue(writer.closed)
+
+    # Test the reader with a 'with' statement.
+    datums = []
+    reader = open(FILENAME, 'rb')
+    datum_reader = io.DatumReader()
+    with datafile.DataFileReader(reader, datum_reader) as dfr:
+      self.assertEquals('foo', dfr.get_meta('test.string'))
+      self.assertEquals('1', dfr.get_meta('test.number'))
+      for datum in dfr:
+        datums.append(datum)
+    self.assertTrue(reader.closed)
+
 if __name__ == '__main__':
   unittest.main()


[prev in list] [next in list] [prev in thread] [next in thread]