[prev in list] [next in list] [prev in thread] [next in thread]
List: hadoop-user
Subject: Re: HBase Mapreduce cannot find Map class
From: Stack <stack () duboce ! net>
Date: 2011-07-28 15:48:13
Message-ID: CADcMMgE-ipSdme4u4QSXJkx+TQ165RR07mpz7dw0eY2QOD9gyA () mail ! gmail ! com
[Download RAW message or body]
See http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/package-summary.html#package_description
for some help.
St.Ack
On Thu, Jul 28, 2011 at 4:04 AM, air <cnweike@gmail.com> wrote:
> ---------- Forwarded message ----------
> From: air <cnweike@gmail.com>
> Date: 2011/7/28
> Subject: HBase Mapreduce cannot find Map class
> To: CDH Users <cdh-user@cloudera.org>
>
>
> import java.io.IOException;
> import java.text.ParseException;
> import java.text.SimpleDateFormat;
> import java.util.Date;
>
> import org.apache.hadoop.conf.Configured;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.hbase.HBaseConfiguration;
> import org.apache.hadoop.hbase.client.HTable;
> import org.apache.hadoop.hbase.client.Put;
> import org.apache.hadoop.hbase.mapred.TableMapReduceUtil;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.Text;
> import org.apache.hadoop.mapred.JobClient;
> import org.apache.hadoop.mapred.JobConf;
> import org.apache.hadoop.mapred.MapReduceBase;
> import org.apache.hadoop.mapred.Mapper;
> import org.apache.hadoop.mapred.OutputCollector;
> import org.apache.hadoop.mapred.Reporter;
> import org.apache.hadoop.mapred.FileInputFormat;
> import org.apache.hadoop.mapred.lib.NullOutputFormat;
> import org.apache.hadoop.util.Tool;
> import org.apache.hadoop.util.ToolRunner;
>
>
> public class LoadToHBase extends Configured implements Tool{
> public static class XMap<K, V> extends MapReduceBase implements
> Mapper<LongWritable, Text, K, V>{
> private JobConf conf;
>
> @Override
> public void configure(JobConf conf){
> this.conf = conf;
> try{
> this.table = new HTable(new HBaseConfiguration(conf),
> "observations");
> }catch(IOException e){
> throw new RuntimeException("Failed HTable construction", e);
> }
> }
>
> @Override
> public void close() throws IOException{
> super.close();
> table.close();
> }
>
> private HTable table;
> public void map(LongWritable key, Text value, OutputCollector
> output, Reporter reporter) throws IOException{
> String[] valuelist = value.toString().split("\t");
> SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd
> HH:mm:ss");
> Date addtime = null; // 用户注册时间
> Date ds = null;
> Long delta_days = null;
> String uid = valuelist[0];
> try {
> addtime = sdf.parse(valuelist[1]);
> } catch (ParseException e) {
> // TODO Auto-generated catch block
> e.printStackTrace();
> }
>
> String ds_str = conf.get("load.hbase.ds", null);
> if (ds_str != null){
> try {
> ds = sdf.parse(ds_str);
> } catch (ParseException e) {
> // TODO Auto-generated catch block
> e.printStackTrace();
> }
> }else{
> ds_str = "2011-07-28";
> }
>
> if (addtime != null && ds != null){
> delta_days = (ds.getTime() - addtime.getTime()) / (24 * 60 *
> 60 * 1000);
> }
>
> if (delta_days != null){
> byte[] rowKey = uid.getBytes();
> Put p = new Put(rowKey);
> p.add("content".getBytes(), "attr1".getBytes(),
> delta_days.toString().getBytes());
> table.put(p);
> }
> }
> }
> /**
> * @param args
> * @throws Exception
> */
> public static void main(String[] args) throws Exception {
> // TODO Auto-generated method stub
> int exitCode = ToolRunner.run(new HBaseConfiguration(), new
> LoadToHBase(), args);
> System.exit(exitCode);
> }
>
> @Override
> public int run(String[] args) throws Exception {
> // TODO Auto-generated method stub
> JobConf conf = new JobConf(getClass());
> TableMapReduceUtil.addDependencyJars(conf);
> FileInputFormat.addInputPath(conf, new Path(args[0]));
> conf.setJobName("LoadToHBase");
> conf.setJarByClass(getClass());
> conf.setMapperClass(XMap.class);
> conf.setNumReduceTasks(0);
> conf.setOutputFormat(NullOutputFormat.class);
> JobClient.runJob(conf);
> return 0;
> }
>
> }
>
> execute it using hbase LoadToHBase /user/hive/warehouse/datamining.db/xxx/
> and it says:
>
> ......
> 11/07/28 17:20:29 INFO mapred.JobClient: Task Id :
> attempt_201107261532_2625_m_000004_1, Status : FAILED
> java.lang.RuntimeException: Error in configuring object
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
> at
> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
> at
> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
> at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387)
> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
> at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127)
> at org.apache.hadoop.mapred.Child.main(Child.java:264)
> Caused by: java.lang.reflect.InvocationTargetException
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
> ... 9 more
> Caused by: java.lang.RuntimeException: java.lang.RuntimeException:
> java.lang.ClassNotFoundException: LoadToHBase$XMap
> at
> org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028)
> at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968)
> at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34)
> ... 14 more
> Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException:
> LoadToHBase$XMap
> at
> org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996)
> at
> org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020)
> ... 16 more
> Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap
> at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
> at java.lang.Class.forName0(Native Method)
> at java.lang.Class.forName(Class.java:247)
> at
> org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943)
> at
> org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994)
> ... 17 more
>
> 11/07/28 17:20:29 INFO mapred.JobClient: Task Id :
> attempt_201107261532_2625_m_000006_1, Status : FAILED
> java.lang.RuntimeException: Error in configuring object
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93)
> at
> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64)
> at
> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
> at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387)
> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
> at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127)
> at org.apache.hadoop.mapred.Child.main(Child.java:264)
> Caused by: java.lang.reflect.InvocationTargetException
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at
> org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88)
> ... 9 more
> Caused by: java.lang.RuntimeException: java.lang.RuntimeException:
> java.lang.ClassNotFoundException: LoadToHBase$XMap
> at
> org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028)
> at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968)
> at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34)
> ... 14 more
> Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException:
> LoadToHBase$XMap
> at
> org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996)
> at
> org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020)
> ... 16 more
> Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap
> at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
> at java.lang.Class.forName0(Native Method)
> at java.lang.Class.forName(Class.java:247)
> at
> org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943)
> at
> org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994)
>
> 11/07/28 17:20:29 INFO mapred.JobClient: Job complete: job_201107261532_2625
> 11/07/28 17:20:29 INFO mapred.JobClient: Counters: 8
> 11/07/28 17:20:29 INFO mapred.JobClient: Job Counters
> 11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=1504081
> 11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all reduces
> waiting after reserving slots (ms)=0
> 11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all maps
> waiting after reserving slots (ms)=0
> 11/07/28 17:20:29 INFO mapred.JobClient: Rack-local map tasks=187
> 11/07/28 17:20:29 INFO mapred.JobClient: Launched map tasks=350
> 11/07/28 17:20:29 INFO mapred.JobClient: Data-local map tasks=163
> 11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=0
> 11/07/28 17:20:29 INFO mapred.JobClient: Failed map tasks=1
> 11/07/28 17:20:29 INFO mapred.JobClient: Job Failed: NA
> Exception in thread "main" java.io.IOException: Job failed!
> at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1246)
> at LoadToHBase.run(LoadToHBase.java:106)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65)
> at LoadToHBase.main(LoadToHBase.java:91)
>
>
> why it cannot find XMap, it's a inner class, it should be found ... I am
> really confused with this situation... thank you for you help
> --
> Knowledge Mangement .
>
>
>
>
> --
> Knowledge Mangement .
>
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic