琐碎-hadoop1.X和2.X的区别

1. jobtracker做了分离，分成了resourceManager和nodemanager；

2. MR变成了和HBase和Hive等一样的yarn上面的一个应用；

3. 1.x的默认块大小为64M，2.x的默认块大小为128M；

4. 在2.x中除了datanode要向namenode报告status，nodemanager也要向ResourceManager报告status

5. MR API差别

旧的WordCount

 1 package org.apache.hadoop.mapred;
 2 
 3 ... ...
 4 
 5 public class WordCount extends Configured implements Tool {
 6   
 7   public static class MapClass extends MapReduceBase
 8     implements Mapper<LongWritable, Text, Text, IntWritable> {
 9     
10     ... ...
11     
12     public void map(LongWritable key, Text value, 
13                     OutputCollector<Text, IntWritable> output, 
14                     Reporter reporter) throws IOException {
15       ... ...
16     }
17   }
18 
19   public static class Reduce extends MapReduceBase
20     implements Reducer<Text, IntWritable, Text, IntWritable> {
21     
22     public void reduce(Text key, Iterator<IntWritable> values,
23                        OutputCollector<Text, IntWritable> output, 
24                        Reporter reporter) throws IOException {
25      ... ...
26     }
27   }
28   
29   static int printUsage() {
30     System.out.println("wordcount [-m <maps>] [-r <reduces>] <input> <output>");
31     ToolRunner.printGenericCommandUsage(System.out);
32     return -1;
33   }
34    
35   public int run(String[] args) throws Exception {
36     ... ...
37     return 0;
38   }
39   public static void main(String[] args) throws Exception {
40     int res = ToolRunner.run(new Configuration(), new WordCount(), args);
41     System.exit(res);
42   }
43 
44 }

新的WordCount

 1 package org.apache.hadoop.examples;
 2 
 3 ... ...
 4 
 5 public class WordCount {
 6 
 7   public static class TokenizerMapper 
 8        extends Mapper<Object, Text, Text, IntWritable>{
 9     
10     ... ... 
11       
12     public void map(Object key, Text value, Context context
13                     ) throws IOException, InterruptedException {
14       ... ...
15     }
16   }
17   
18   public static class IntSumReducer 
19        extends Reducer<Text,IntWritable,Text,IntWritable> {
20     private IntWritable result = new IntWritable();
21 
22     public void reduce(Text key, Iterable<IntWritable> values, 
23                        Context context
24                        ) throws IOException, InterruptedException {
25       ... ...
26     }
27   }
28 
29   public static void main(String[] args) throws Exception {
30     ... ...
31     System.exit(job.waitForCompletion(true) ? 0 : 1);
32   }
33 }

欲为大树，何与草争；心若不动，风又奈何。