'Keys in TableReducer is empty when map reads from the HDFS'

[prev in list] [next in list] [prev in thread] [next in thread] 

List:       hadoop-user
Subject:    Keys in TableReducer is empty when map reads from the HDFS
From:       Konstantin Zhukov <kzhukov () thumbtack ! net>
Date:       2015-10-27 9:31:38
Message-ID: FF67E826-09A3-48E3-9F30-87939B81CDA4 () thumbtack ! net
[Download RAW message or body]

Could you please give me advise how I can resolve my problem?

Now I read file from the hdfs and it is ok. I log values before write it in context \
and it is right. But if I use TableReducer I get empty keys.

public class WordNetJob {
    private static final Logger log = Logger.getLogger(WordNetJob.class);

    public static void main(String[] args) throws Exception {
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum", "localhost");

        Job job = Job.getInstance(conf, "WordNet");

        job.setJarByClass(WordNetJob.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setMapperClass(WordNetMapper.class);

        TableMapReduceUtil.initTableReducerJob("Neighbors", WordNetReducer.class, \
job);  job.setReducerClass(WordNetReducer.class);

        FileInputFormat.setInputPaths(job, new Path("input"));
        job.waitForCompletion(true);
    }

    public static class WordNetMapper extends Mapper<LongWritable, Text, Text, Text> \
                {
        private static final Logger log = Logger.getLogger(WordNetMapper.class);

        @Override
        protected void map(LongWritable key, Text value, Context context) throws \
IOException, InterruptedException {  //...
            log.info <http://log.info/>("Key: [" + new Text(currentWord).toString() + \
"]; Value: [" + new Text(previousWord).toString() + "]"); //it is good  \
context.write(new Text(previousWord), new Text(currentWord));  }
    }

    public static class WordNetReducer extends TableReducer<Text, Text, \
                ImmutableBytesWritable> {
        private static final Logger log = Logger.getLogger(WordNetReducer.class);

        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) \
throws IOException,  InterruptedException {
            //..
            log.info <http://log.info/>("ReduceKey: " + key.toString()); //key is \
empty now  //..
        }
    }
}


[Attachment #3 (unknown)]

<html><head><meta http-equiv="Content-Type" content="text/html \
charset=us-ascii"><meta http-equiv="Content-Type" content="text/html \
charset=us-ascii"><meta http-equiv="Content-Type" content="text/html \
charset=us-ascii"></head><body style="word-wrap: break-word; -webkit-nbsp-mode: \
space; -webkit-line-break: after-white-space;" class="">Could you please give me \
advise how I can resolve my problem?<div class=""><br class=""></div><div \
class="">Now I read file from the hdfs and it is ok. I log values before write it in \
context and it is right. But if I use TableReducer I get empty keys.</div><div \
class=""><br class=""></div><div class=""><pre style="margin-top: 0px; padding: 5px; \
border: 0px; font-size: 13px; overflow: auto; width: auto; max-height: 600px; \
font-family: Consolas, Menlo, Monaco, 'Lucida Console', 'Liberation Mono', 'DejaVu \
Sans Mono', 'Bitstream Vera Sans Mono', 'Courier New', monospace, sans-serif; \
word-wrap: normal; color: rgb(34, 34, 34); widows: 1; background-color: rgb(238, 238, \
238);" class=""><code style="margin: 0px; padding: 0px; border: 0px; font-family: \
Consolas, Menlo, Monaco, 'Lucida Console', 'Liberation Mono', 'DejaVu Sans Mono', \
'Bitstream Vera Sans Mono', 'Courier New', monospace, sans-serif; white-space: \
inherit;" class="">public class WordNetJob {  private static final Logger log = \
Logger.getLogger(WordNetJob.class);

    public static void main(String[] args) throws Exception {
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum", "localhost");

        Job job = Job.getInstance(conf, "WordNet");

        job.setJarByClass(WordNetJob.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setMapperClass(WordNetMapper.class);

        TableMapReduceUtil.initTableReducerJob("Neighbors", WordNetReducer.class, \
job);  job.setReducerClass(WordNetReducer.class);

        FileInputFormat.setInputPaths(job, new Path("input"));
        job.waitForCompletion(true);
    }

    public static class WordNetMapper extends Mapper&lt;LongWritable, Text, Text, \
                Text&gt; {
        private static final Logger log = Logger.getLogger(WordNetMapper.class);

        @Override
        protected void map(LongWritable key, Text value, Context context) throws \
IOException, InterruptedException {  //...
            <a href="http://log.info" class="">log.info</a>("Key: [" + new \
Text(currentWord).toString() + "]; Value: [" + new Text(previousWord).toString() + \
"]"); //it is good  context.write(new Text(previousWord), new Text(currentWord));
        }
    }

    public static class WordNetReducer extends TableReducer&lt;Text, Text, \
                ImmutableBytesWritable&gt; {
        private static final Logger log = Logger.getLogger(WordNetReducer.class);

        @Override
        protected void reduce(Text key, Iterable&lt;Text&gt; values, Context context) \
throws IOException,  InterruptedException {
            //..
            <a href="http://log.info" class="">log.info</a>("ReduceKey: " + \
key.toString()); //key is empty now  //..
        }
    }
}</code></pre><div class=""><br class=""></div></div><div class=""><br \
class=""></div><div class=""><br class=""></div></body></html>



[prev in list] [next in list] [prev in thread] [next in thread]
Configure | About | News | Add a list | Sponsored by KoreLogic