'RE: java.lang.RuntimeException: java.io.EOFException at'

[prev in list] [next in list] [prev in thread] [next in thread] 

List:       hadoop-user
Subject:    RE: java.lang.RuntimeException: java.io.EOFException at
From:       Tali K <ncherryus () hotmail ! com>
Date:       2010-09-30 19:15:28
Message-ID: SNT135-w5297EC1DBC1B8AC96A55FAB0680 () phx ! gbl
[Download RAW message or body]


I have only 2 Strings, no lists or primitives, and I am not concatenating strings.
So all my class has is :  String a;
                              String b;
 

 
> Date: Thu, 30 Sep 2010 10:42:38 -0700
> Subject: Re: java.lang.RuntimeException: java.io.EOFException at \
>                 org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103)
>                 
> From: yuzhihong@gmail.com
> To: common-user@hadoop.apache.org
> 
> Line 84 is empty.
> Line 83 is:
> out.writeUTF(query_id);
> 
> Please send the stack trace that corresponds to your attachment.
> 
> From previous discussion:
> In the very begining of readFields(), clear all available fields (lists,
> primitives, etc).
> The best way to to do that is to create a clearFields() function, that will
> be called both from "readFields()" and from the empty constructor.
> 
> On Thu, Sep 30, 2010 at 10:16 AM, Tali K <ncherryus@hotmail.com> wrote:
> 
> > *You are right, there is no readInt , I have only 2 String fields in
> > MsRead
> > Here are lines: I'll also will send both files in attch.
> > Thanks in advance for your help.*
> > 
> > @Override
> > public void readFields(DataInput in) throws IOException {
> > 
> > query_id = in.readUTF();
> > record = in.readUTF();
> > 
> > }
> > @Override
> > public void write(DataOutput out) throws IOException {
> > out.writeUTF(query_id);
> > 
> > out.writeUTF(record);
> > 
> > }
> > 
> > 
> > 
> > 
> > 
> > public static class FirstComparator extends WritableComparator {
> > 
> > private static final Text.Comparator TEXT_COMPARATOR = new
> > Text.Comparator();
> > 
> > public FirstComparator() {
> > super(MsRead.class);
> > }
> > 
> > @Override
> > public int compare(byte[] b1, int s1, int l1,
> > byte[] b2, int s2, int l2) {
> > 
> > try {
> > int firstL1 = WritableUtils.decodeVIntSize(b1[s1]) +
> > readVInt(b1, s1);
> > int firstL2 = WritableUtils.decodeVIntSize(b2[s2]) +
> > readVInt(b2, s2);
> > return TEXT_COMPARATOR.compare(b1, s1, firstL1, b2, s2,
> > firstL2);
> > } catch (IOException e) {
> > throw new IllegalArgumentException(e);
> > }
> > }
> > 
> > @Override
> > public int compare(WritableComparable a, WritableComparable b) {
> > if (a instanceof MsRead && b instanceof MsRead) {
> > 
> > //System.err.println("COMPARE " + ((MsRead)a).getType() + "\t"
> > + ((MsRead)b).getType() + "\t"
> > // + (((MsRead) a).toString().compareTo(((MsRead)
> > b).toString())));
> > return (((MsRead) a).toString().compareTo(((MsRead)
> > b).toString()));
> > 
> > }
> > return super.compare(a, b);
> > }
> > 
> > 
> > }
> > 
> > @Override
> > public int compareTo(MsRead o) {
> > return this.toString().compareTo(o.toString());
> > }
> > @Override
> > public boolean equals(Object right) {
> > if (right instanceof MsRead )
> > {
> > return (query_id.equals(((MsRead)right).query_id));
> > }
> > else
> > return false;
> > }
> > @Override
> > public int hashCode() {
> > return query_id.hashCode() ;
> > }
> > 
> > > Date: Wed, 29 Sep 2010 22:27:15 -0700
> > > Subject: Re: java.lang.RuntimeException: java.io.EOFException at
> > org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103)
> > > From: yuzhihong@gmail.com
> > > To: common-user@hadoop.apache.org
> > 
> > > 
> > > Your MsRead.readFields() doesn't contain readInt().
> > > Can you show us the lines around line 84 of MsRead.java ?
> > > 
> > > On Wed, Sep 29, 2010 at 2:44 PM, Tali K <ncherryus@hotmail.com> wrote:
> > > 
> > > > 
> > > > HI All,
> > > > 
> > > > I am getting this Exception on a cluster(10 nodes) when I am running
> > > > simple hadoop map / reduce job.
> > > > I don't have this Exception while running it on my desktop in hadoop's
> > > > pseudo distributed mode.
> > > > Can somebody help? I would really appreciate it.
> > > > 
> > > > 
> > > > 10/09/29 14:28:34 INFO mapred.JobClient: map 100% reduce 30%
> > > > 10/09/29 14:28:36 INFO mapred.JobClient: Task Id :
> > > > attempt_201009291306_0004_r_000000_0, Status : FAILED
> > > > java.lang.RuntimeException: java.io.EOFException
> > > > at
> > > > 
> > org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:103)
> > > > at
> > > > org.apache.hadoop.mapred.Merger$MergeQueue.lessThan(Merger.java:373)
> > > > at
> > > > org.apache.hadoop.util.PriorityQueue.upHeap(PriorityQueue.java:123)
> > > > at org.apache.hadoop.util.PriorityQueue.put(PriorityQueue.java:50)
> > > > at org.apache.hadoop.mapred.Merger$MergeQueue.merge(Merger.java:447)
> > > > at org.apache.hadoop.mapred.Merger$MergeQueue.merge(Merger.java:381)
> > > > at org.apache.hadoop.mapred.Merger.merge(Merger.java:107)
> > > > at org.apache.hadoop.mapred.Merger.merge(Merger.java:93)
> > > > at
> > > > 
> > org.apache.hadoop.mapred.ReduceTask$ReduceCopier.createKVIterator(ReduceTask.java:2316)
> > 
> > > > at
> > > > 
> > org.apache.hadoop.mapred.ReduceTask$ReduceCopier.access$400(ReduceTask.java:576)
> > > > at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
> > > > at org.apache.hadoop.mapred.Child.main(Child.java:170)
> > > > Caused by: java.io.EOFException
> > > > at java.io.DataInputStream.readInt(DataInputStream.java:375)
> > > > at speeditup.MsRead.readFields(MsRead.java:84)
> > > > at
> > > > 
> > org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java:97)
> > > > ... 11 more
> > > > Here is a class that has WritableComparator.compare. It has only 2
> > strings
> > > > max length 20 characters for each.
> > > > 
> > > > public class MsRead implements WritableComparable < MsRead> {
> > > > private static final Log LOG =
> > > > LogFactory.getLog(speeditup.CalculateMinEvalue.class);
> > > > 
> > > > private String query_id;
> > > > 
> > > > private String record;
> > > > 
> > > > 
> > > > 
> > > > public String getRecord() {
> > > > return record;
> > > > }
> > > > public void setRecord(String record) {
> > > > this.record = record;
> > > > }
> > > > 
> > > > public String getQuery_id() {
> > > > return query_id;
> > > > }
> > > > 
> > > > public void setQuery_id(String queryId) {
> > > > query_id = queryId;
> > > > }
> > > > 
> > > > public MsRead()
> > > > {
> > > > ;
> > > > }
> > > > public MsRead(String a, String r)
> > > > {
> > > > setQuery_id(a);
> > > > 
> > > > setRecord(r);
> > > > }
> > > > 
> > > > 
> > > > 
> > > > 
> > > > 
> > > > @Override
> > > > public void readFields(DataInput in) throws IOException {
> > > > LOG.debug("******myreadFields" + " " );
> > > > LOG.warn("******myreadFields" + " " );
> > > > LOG.info("******myreadFields" + " " );
> > > > query_id = in.readUTF();
> > > > record = in.readUTF();
> > > > 
> > > > }
> > > > @Override
> > > > public void write(DataOutput out) throws IOException {
> > > > out.writeUTF(query_id);
> > > > out.writeUTF(record);
> > > > 
> > > > }
> > > > 
> > > > 
> > > > 
> > > > 
> > > > 
> > > > public static class FirstComparator extends WritableComparator {
> > > > 
> > > > private static final Text.Comparator TEXT_COMPARATOR = new
> > > > Text.Comparator();
> > > > 
> > > > public FirstComparator() {
> > > > super(MsRead.class);
> > > > }
> > > > 
> > > > @Override
> > > > public int compare(byte[] b1, int s1, int l1,
> > > > byte[] b2, int s2, int l2) {
> > > > 
> > > > try {
> > > > int firstL1 = WritableUtils.decodeVIntSize(b1[s1]) +
> > > > readVInt(b1, s1);
> > > > int firstL2 = WritableUtils.decodeVIntSize(b2[s2]) +
> > > > readVInt(b2, s2);
> > > > return TEXT_COMPARATOR.compare(b1, s1, firstL1, b2, s2,
> > > > firstL2);
> > > > } catch (IOException e) {
> > > > throw new IllegalArgumentException(e);
> > > > }
> > > > 
> > > > }
> > > > 
> > > > @Override
> > > > public int compare(WritableComparable a, WritableComparable b) {
> > > > if (a instanceof MsRead && b instanceof MsRead) {
> > > > 
> > > > //System.err.println("COMPARE " + ((MsRead)a).getType() + "\t" +
> > > > ((MsRead)b).getType() + "\t"
> > > > // + (((MsRead) a).toString().compareTo(((MsRead)
> > > > b).toString())));
> > > > return (((MsRead) a).toString().compareTo(((MsRead)
> > > > b).toString()));
> > > > 
> > > > }
> > > > return super.compare(a, b);
> > > > }
> > > > 
> > > > 
> > > > }
> > > > 
> > > > @Override
> > > > public int compareTo(MsRead o) {
> > > > return this.toString().compareTo(o.toString());
> > > > }
> > > > @Override
> > > > public boolean equals(Object right) {
> > > > if (right instanceof MsRead )
> > > > {
> > > > return (query_id.equals(((MsRead)right).query_id));
> > > > }
> > > > else
> > > > return false;
> > > > }
> > > > @Override
> > > > public int hashCode() {
> > > > return query_id.hashCode() ;
> > > > }
> > > > 
> > > > @Override
> > > > public String toString()
> > > > {
> > > > return query_id;
> > > > }
> > > > public String toOutputString()
> > > > {
> > > > return record;
> > > > }
> > > > 
> > > > }
> > > > 
> > 
 		 	   		  



[prev in list] [next in list] [prev in thread] [next in thread]
Configure | About | News | Add a list | Sponsored by KoreLogic