[prev in list] [next in list] [prev in thread] [next in thread]
List: avro-commits
Subject: svn commit: r1480808 - in /avro/trunk: ./ lang/java/avro/src/main/java/org/apache/avro/
From: cutting () apache ! org
Date: 2013-05-09 22:15:17
Message-ID: 20130509221518.8247323888E4 () eris ! apache ! org
[Download RAW message or body]
Author: cutting
Date: Thu May 9 22:15:16 2013
New Revision: 1480808
URL: http://svn.apache.org/r1480808
Log:
AVRO-1316. Java: Fix compiler to split long schema string constants so javac can \
compile them. Contributed by Jeremy Kahn.
Added:
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java (with \
props) Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java
avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm
avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu May 9 22:15:16 2013
@@ -80,6 +80,9 @@ Trunk (not yet released)
AVRO-1322. Java: Add Hadoop version classifier to trevni-avro
Maven artifacts. (massie)
+ AVRO-1316. Java: Fix compiler to split long schema string
+ constants so javac can compile them. (Jeremy Kahn via cutting)
+
Avro 1.7.4 (22 February 2012)
NEW FEATURES
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java Thu May 9 \
22:15:16 2013 @@ -366,6 +366,14 @@ public class Protocol extends JsonProper
return parse(Schema.FACTORY.createJsonParser(stream));
}
+ /** Read a protocol from one or more json strings */
+ public static Protocol parse(String string, String... more) {
+ StringBuilder b = new StringBuilder(string);
+ for (String part : more)
+ b.append(part);
+ return parse(b.toString());
+ }
+
/** Read a protocol from a Json string. */
public static Protocol parse(String string) {
try {
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java Thu May 9 \
22:15:16 2013 @@ -915,6 +915,14 @@ public abstract class Schema extends Jso
return parse(FACTORY.createJsonParser(in));
}
+ /** Read a schema from one or more json strings */
+ public Schema parse(String s, String... more) {
+ StringBuilder b = new StringBuilder(s);
+ for (String part : more)
+ b.append(part);
+ return parse(b.toString());
+ }
+
/** Parse a schema from the provided string.
* If named, the schema is added to the names known to this parser. */
public Schema parse(String s) {
Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java \
(original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java Thu May \
9 22:15:16 2013 @@ -17,8 +17,11 @@
*/
package org.apache.avro;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+import org.junit.Test;
public class TestProtocol {
@@ -29,5 +32,26 @@ public class TestProtocol {
p2.addProp("a","2");
assertFalse(p1.equals(p2));
}
+
+ @Test
+ public void testSplitProtocolBuild() {
+ Protocol p = new Protocol("P", null, "foo");
+ p.addProp("property", "some value");
+
+ String protocolString = p.toString();
+ final int mid = protocolString.length() / 2;
+ String[] parts = {
+ protocolString.substring(0, mid),
+ protocolString.substring(mid),
+ };
+
+ Protocol parsedStringProtocol = org.apache.avro.Protocol.parse(protocolString);
+ Protocol parsedArrayOfStringProtocol =
+ org.apache.avro.Protocol.parse(protocolString.substring(0, mid),
+ protocolString.substring(mid));
+ assertNotNull(parsedStringProtocol);
+ assertNotNull(parsedArrayOfStringProtocol);
+ assertEquals(parsedStringProtocol.toString(), \
parsedArrayOfStringProtocol.toString()); + }
}
Added: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java?rev=1480808&view=auto
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java (added)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java Thu May \
9 22:15:16 2013 @@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.junit.Test;
+
+public class TestSchema {
+ @Test
+ public void testSplitSchemaBuild() {
+ Schema s = SchemaBuilder
+ .recordType("HandshakeRequest")
+ .namespace("org.apache.avro.ipc")
+ .unionType("clientProtocol", SchemaBuilder.unionType(
+ SchemaBuilder.NULL,
+ SchemaBuilder.STRING)
+ .build())
+ .unionType("meta", SchemaBuilder.unionType(
+ SchemaBuilder.NULL,
+ SchemaBuilder.mapType(SchemaBuilder.BYTES)
+ .build())
+ .build())
+ .build();
+
+ String schemaString = s.toString();
+ final int mid = schemaString.length() / 2;
+
+ Schema parsedStringSchema = new \
org.apache.avro.Schema.Parser().parse(s.toString()); + Schema \
parsedArrayOfStringSchema = + new org.apache.avro.Schema.Parser().parse
+ (schemaString.substring(0, mid), schemaString.substring(mid));
+ assertNotNull(parsedStringSchema);
+ assertNotNull(parsedArrayOfStringSchema);
+ assertEquals(parsedStringSchema.toString(), \
parsedArrayOfStringSchema.toString()); + }
+}
Propchange: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/java/org/apac \
he/avro/compiler/specific/SpecificCompiler.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java \
(original)
+++ avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java \
Thu May 9 22:15:16 2013 @@ -548,6 +548,25 @@ public class SpecificCompiler {
return new String[0];
}
+ // maximum size for string constants, to avoid javac limits
+ int maxStringChars = 8192;
+
+ /** Utility for template use. Takes a (potentially overly long) string and
+ * splits it into a quoted, comma-separted sequence of escaped strings.
+ * @param s The string to split
+ * @return A sequence of quoted, comma-separated, escaped strings
+ */
+ public String javaSplit(String s) throws IOException {
+ StringBuilder b = new StringBuilder("\""); // initial quote
+ for (int i = 0; i < s.length(); i += maxStringChars) {
+ if (i != 0) b.append("\",\""); // insert quote-comma-quote
+ String chunk = s.substring(i, Math.min(s.length(), i + maxStringChars));
+ b.append(javaEscape(chunk)); // escape chunks
+ }
+ b.append("\""); // final quote
+ return b.toString();
+ }
+
/** Utility for template use. Escapes quotes and backslashes. */
public static String javaEscape(Object o) {
return o.toString().replace("\\","\\\\").replace("\"", "\\\"");
Modified: avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/velocity/org/ \
apache/avro/compiler/specific/templates/java/classic/protocol.vm?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm \
(original)
+++ avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm \
Thu May 9 22:15:16 2013 @@ -28,7 +28,7 @@ package $protocol.getNamespace();
#end
@org.apache.avro.specific.AvroGenerated
public interface $this.mangle($protocol.getName()) {
- public static final org.apache.avro.Protocol PROTOCOL = \
org.apache.avro.Protocol.parse("${this.javaEscape($protocol.toString())}"); + public \
static final org.apache.avro.Protocol PROTOCOL = \
org.apache.avro.Protocol.parse(${this.javaSplit($protocol.toString())}); #foreach \
($e in $protocol.getMessages().entrySet()) #set ($name = $e.getKey())
#set ($message = $e.getValue())
Modified: avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/velocity/org/ \
apache/avro/compiler/specific/templates/java/classic/record.vm?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm \
(original)
+++ avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm \
Thu May 9 22:15:16 2013 @@ -27,7 +27,7 @@ package $schema.getNamespace();
#end
@org.apache.avro.specific.AvroGenerated
public class ${this.mangle($schema.getName())}#if ($schema.isError()) extends \
org.apache.avro.specific.SpecificExceptionBase#else extends \
org.apache.avro.specific.SpecificRecordBase#end implements \
org.apache.avro.specific.SpecificRecord {
- public static final org.apache.avro.Schema SCHEMA$ = new \
org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}"); + \
public static final org.apache.avro.Schema SCHEMA$ = new \
org.apache.avro.Schema.Parser().parse(${this.javaSplit($schema.toString())}); public \
static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } #foreach ($field \
in $schema.getFields()) #if ($field.doc())
Modified: avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/ipc/src/test/java/org/apache/av \
ro/compiler/specific/TestSpecificCompiler.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java \
(original)
+++ avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java \
Thu May 9 22:15:16 2013 @@ -123,9 +123,7 @@ public class TestSpecificCompiler {
}
- @Test
- public void testManglingForRecords() throws IOException {
- String schema = "" +
+ private static String SCHEMA =
"{ \"name\": \"volatile\", \"type\": \"record\", " +
" \"fields\": [ {\"name\": \"package\", \"type\": \"string\" }," +
" {\"name\": \"data\", \"type\": \"int\" }," +
@@ -133,8 +131,12 @@ public class TestSpecificCompiler {
" {\"name\": \"defaultValue\", \"type\": \"int\" }," +
" {\"name\": \"other\", \"type\": \"int\" }," +
" {\"name\": \"short\", \"type\": \"volatile\" } ] }";
+
+
+ @Test
+ public void testManglingForRecords() throws IOException {
Collection<OutputFile> c =
- new SpecificCompiler(Schema.parse(schema)).compile();
+ new SpecificCompiler(Schema.parse(SCHEMA)).compile();
assertEquals(1, c.size());
String contents = c.iterator().next().contents;
@@ -161,6 +163,22 @@ public class TestSpecificCompiler {
}
@Test
+ public void testSchemaSplit() throws IOException {
+ SpecificCompiler compiler = new SpecificCompiler(Schema.parse(SCHEMA));
+ compiler.maxStringChars = 10;
+ Collection<OutputFile> files = compiler.compile();
+ assertCompilesWithJavaCompiler(files);
+ }
+
+ @Test
+ public void testProtocolSplit() throws IOException {
+ SpecificCompiler compiler = new SpecificCompiler(Protocol.parse(PROTOCOL));
+ compiler.maxStringChars = 10;
+ Collection<OutputFile> files = compiler.compile();
+ assertCompilesWithJavaCompiler(files);
+ }
+
+ @Test
public void testSchemaWithDocs() {
Collection<OutputFile> outputs = new SpecificCompiler(
Schema.parse(TestSchema.SCHEMA_WITH_DOC_TAGS)).compile();
Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/ \
avro/tool/TestSpecificCompilerTool.java?rev=1480808&r1=1480807&r2=1480808&view=diff \
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java \
(original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java \
Thu May 9 22:15:16 2013 @@ -48,6 +48,15 @@ public class TestSpecificCompilerTool {
private static final File TEST_EXPECTED_PLAYER =
new File(TEST_EXPECTED_OUTPUT_DIR, "Player.java");
+ private static final File TEST_EXPECTED_STRING_OUTPUT_DIR =
+ new File(TEST_DIR, "output-string");
+ private static final File TEST_EXPECTED_STRING_POSITION =
+ new File(TEST_EXPECTED_STRING_OUTPUT_DIR,
+ "avro/examples/baseball/Position.java");
+ private static final File TEST_EXPECTED_STRING_PLAYER =
+ new File(TEST_EXPECTED_STRING_OUTPUT_DIR,
+ "avro/examples/baseball/Player.java");
+
// where test output goes
private static final File TEST_OUTPUT_DIR =
new File("target/compiler/output");
@@ -59,9 +68,9 @@ public class TestSpecificCompilerTool {
private static final File TEST_OUTPUT_STRING_DIR =
new File("target/compiler/output-string");
private static final File TEST_OUTPUT_STRING_PLAYER =
- new File(TEST_OUTPUT_DIR, "avro/examples/baseball/Player.java");
+ new File(TEST_OUTPUT_STRING_DIR, "avro/examples/baseball/Player.java");
private static final File TEST_OUTPUT_STRING_POSITION =
- new File(TEST_OUTPUT_DIR, "avro/examples/baseball/Position.java");
+ new File(TEST_OUTPUT_STRING_DIR, "avro/examples/baseball/Position.java");
@Before
public void setUp() {
@@ -105,9 +114,9 @@ public class TestSpecificCompilerTool {
doCompile(new String[]{"-string", "schema",
TEST_INPUT_DIR.toString() + "/position.avsc",
TEST_INPUT_DIR.toString() + "/player.avsc",
- TEST_OUTPUT_DIR.getPath()});
- assertFileMatch(TEST_EXPECTED_POSITION, TEST_OUTPUT_STRING_POSITION);
- assertFileMatch(TEST_EXPECTED_PLAYER, TEST_OUTPUT_STRING_PLAYER);
+ TEST_OUTPUT_STRING_DIR.getPath()});
+ assertFileMatch(TEST_EXPECTED_STRING_POSITION, TEST_OUTPUT_STRING_POSITION);
+ assertFileMatch(TEST_EXPECTED_STRING_PLAYER, TEST_OUTPUT_STRING_PLAYER);
}
// Runs the actual compiler tool with the given input args
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic