MapReduce 이전 API -지도에 명령 줄 인수 전달

Question

이전 API를 사용하여 HDFS에 저장된 입력 파일에서 검색 문자열 (명령 줄 인수를 통해 전달 된)의 발생을 찾기위한 MapReduce 작업을 코딩하고 있습니다.

다음

public class StringSearchDriver 
{ 

    public static void main(String[] args) throws IOException 
    { 
     JobConf jc = new JobConf(StringSearchDriver.class); 
     jc.set("SearchWord", args[2]); 
     jc.setJobName("String Search"); 
     FileInputFormat.addInputPath(jc, new Path(args[0])); 
     FileOutputFormat.setOutputPath(jc, new Path(args[1])); 
     jc.setMapperClass(StringSearchMap.class); 
     jc.setReducerClass(StringSearchReduce.class); 
     jc.setOutputKeyClass(Text.class); 
     jc.setOutputValueClass(IntWritable.class); 
     JobClient.runJob(jc); 
    } 
} 

내 매퍼 클래스입니다 - - 아래

내 드라이버 클래스입니다 ("안녕하세요"전달 된 명령 줄 문자열)

public class StringSearchMap extends MapReduceBase implements 
     Mapper<LongWritable, Text, Text, IntWritable> 
{ 
    String searchWord; 

    public void configure(JobConf jc) 
    { 
     searchWord = jc.get("SearchWord"); 

    } 



    @Override 
    public void map(LongWritable key, Text value, 
      OutputCollector<Text, IntWritable> out, Reporter reporter) 
      throws IOException 
    { 
     String[] input = value.toString().split(""); 

     for(String word:input) 
     { 
      if (word.equalsIgnoreCase(searchWord)) 
       out.collect(new Text(word), new IntWritable(1)); 
     } 
    } 

} 

작업을 실행에, 나는 아래에 무엇입니까 오류 -

14/09/21 22:35:41 INFO mapred.JobClient: Task Id : attempt_201409212134_0005_m_000001_2, Status : FAILED 
java.lang.ClassCastException: interface javax.xml.soap.Text 
    at java.lang.Class.asSubclass(Class.java:3129) 
    at org.apache.hadoop.mapred.JobConf.getOutputKeyComparator(JobConf.java:795) 
    at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.<init>(MapTask.java:964) 
    at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:422) 
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:366) 
    at org.apache.hadoop.mapred.Child$4.run(Child.java:255) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Subject.java:416) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190) 
    at org.apache.hadoop.mapred.Child.main(Child.java:249) 

제안 해주십시오.

Answer 1

하면 자동 잘못된 가져 오기를 수입을 확인하십시오. 대신 수입 org.apache.hadoop.io.Text 당신 수입 당신이 blog에서 샘플 잘못 가져 오기를 찾을 수 있습니다

javax.xml.soap.Text도.

한 점은, 새로운 API이 작동

import java.io.IOException; 
import java.util.StringTokenizer; 

import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.conf.Configured; 
import org.apache.hadoop.fs.FileSystem; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.IntWritable; 
import org.apache.hadoop.io.LongWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.Mapper; 
import org.apache.hadoop.mapreduce.Reducer; 
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; 
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 
import org.apache.hadoop.util.Tool; 
import org.apache.hadoop.util.ToolRunner; 

/** 
* @author Unmesha sreeveni 
* @Date 23 sep 2014 
*/ 
public class StringSearchDriver extends Configured implements Tool { 
    public static class Map extends 
    Mapper<LongWritable, Text, Text, IntWritable> { 

     private final static IntWritable one = new IntWritable(1); 
     private Text word = new Text(); 

     public void map(LongWritable key, Text value, Context context) 
       throws IOException, InterruptedException { 
      Configuration conf = context.getConfiguration(); 
      String line = value.toString(); 
      String searchString = conf.get("word"); 
      StringTokenizer tokenizer = new StringTokenizer(line); 
      while (tokenizer.hasMoreTokens()) { 
       String token = tokenizer.nextToken(); 
       if(token.equals(searchString)){ 
        word.set(token); 
        context.write(word, one); 
       } 

      } 
     } 
    } 

    public static class Reduce extends 
    Reducer<Text, IntWritable, Text, IntWritable> { 

     public void reduce(Text key, Iterable<IntWritable> values, 
       Context context) throws IOException, InterruptedException { 

      int sum = 0; 
      for (IntWritable val : values) { 
       sum += val.get(); 
      } 
      context.write(key, new IntWritable(sum)); 
     } 
    } 
    public static void main(String[] args) throws Exception { 
     Configuration conf = new Configuration(); 
     int res = ToolRunner.run(conf, new StringSearchDriver(), args); 
     System.exit(res); 

    } 
    @Override 
    public int run(String[] args) throws Exception { 
     // TODO Auto-generated method stub 
     if (args.length != 3) { 
      System.out 
      .printf("Usage: Search String <input dir> <output dir> <search word> \n"); 
      System.exit(-1); 
     } 

     String source = args[0]; 
     String dest = args[1]; 
     String searchword = args[2]; 
     Configuration conf = new Configuration(); 
     conf.set("word", searchword); 
     Job job = new Job(conf, "Search String"); 
     job.setJarByClass(StringSearchDriver.class); 
     FileSystem fs = FileSystem.get(conf); 

     Path in =new Path(source); 
     Path out =new Path(dest); 
     if (fs.exists(out)) { 
      fs.delete(out, true); 
     } 

     job.setMapOutputKeyClass(Text.class); 
     job.setMapOutputValueClass(IntWritable.class); 
     job.setOutputKeyClass(Text.class); 
     job.setOutputValueClass(IntWritable.class); 
     job.setMapperClass(Map.class); 
     job.setReducerClass(Reduce.class); 
     job.setInputFormatClass(TextInputFormat.class); 
     job.setOutputFormatClass(TextOutputFormat.class); 
     FileInputFormat.addInputPath(job, in); 
     FileOutputFormat.setOutputPath(job, out); 
     boolean sucess = job.waitForCompletion(true); 
     return (sucess ? 0 : 1); 
    } 
} 

에게 새로운 API

편집

내가 사용을 채택하는 것이 좋습니다.

Answer 2

텍스트; 필요한 hadoop 패키지는 org.apache.hadoop.io입니다.. 은 패키지

import java.io.IOException; 
import java.util.*; 

import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.conf.*; 
import org.apache.hadoop.io.*; 
import org.apache.hadoop.mapred.*; 
import org.apache.hadoop.util.*;