开发+运行第一个Mahout的程序
代码：
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package chen.test.kmeans; import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.ToolRunner; import org.apache.mahout.clustering.Cluster; import org.apache.mahout.clustering.canopy.CanopyDriver; import org.apache.mahout.clustering.conversion.InputDriver; import org.apache.mahout.clustering.kmeans.KMeansDriver; import org.apache.mahout.clustering.kmeans.RandomSeedGenerator; import org.apache.mahout.common.AbstractJob; import org.apache.mahout.common.ClassUtils; import org.apache.mahout.common.HadoopUtil; import org.apache.mahout.common.commandline.DefaultOptionCreator; import org.apache.mahout.common.distance.DistanceMeasure; import org.apache.mahout.common.distance.EuclideanDistanceMeasure; import org.apache.mahout.common.distance.SquaredEuclideanDistanceMeasure; import org.apache.mahout.utils.clustering.ClusterDumper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public final class TwoJob extends AbstractJob { private static final Logger log = LoggerFactory.getLogger(TwoJob.class); private static final String DIRECTORY_CONTAINING_CONVERTED_INPUT = "data"; private TwoJob() { } public static void main(String[] args) throws Exception { if (args.length > 0) { log.info("Running with only user-supplied arguments"); ToolRunner.run(new Configuration(), new TwoJob(), args); } else { log.info("Running with default arguments"); Path output = new Path("output"); Configuration conf = new Configuration(); HadoopUtil.delete(conf, output); run(conf, new Path("testdata"), output, new EuclideanDistanceMeasure(), 2, 0.5, 10); } } @Override public int run(String[] args) throws Exception { addInputOption(); addOutputOption(); addOption(DefaultOptionCreator.distanceMeasureOption().create()); addOption(DefaultOptionCreator.numClustersOption().create()); addOption(DefaultOptionCreator.t1Option().create()); addOption(DefaultOptionCreator.t2Option().create()); addOption(DefaultOptionCreator.convergenceOption().create()); addOption(DefaultOptionCreator.maxIterationsOption().create()); addOption(DefaultOptionCreator.overwriteOption().create()); Map<String,List<String>> argMap = parseArguments(args); if (argMap == null) { return -1; } Path input = getInputPath(); Path output = getOutputPath(); String measureClass = getOption(DefaultOptionCreator.DISTANCE_MEASURE_OPTION); if (measureClass == null) { measureClass = SquaredEuclideanDistanceMeasure.class.getName(); } double convergenceDelta = Double.parseDouble(getOption(DefaultOptionCreator.CONVERGENCE_DELTA_OPTION)); int maxIterations = Integer.parseInt(getOption(DefaultOptionCreator.MAX_ITERATIONS_OPTION)); if (hasOption(DefaultOptionCreator.OVERWRITE_OPTION)) { HadoopUtil.delete(getConf(), output); } DistanceMeasure measure = ClassUtils.instantiateAs(measureClass, DistanceMeasure.class); if (hasOption(DefaultOptionCreator.NUM_CLUSTERS_OPTION)) { int k = Integer.parseInt(getOption(DefaultOptionCreator.NUM_CLUSTERS_OPTION)); run(getConf(), input, output, measure, k, convergenceDelta, maxIterations); } else { double t1 = Double.parseDouble(getOption(DefaultOptionCreator.T1_OPTION)); double t2 = Double.parseDouble(getOption(DefaultOptionCreator.T2_OPTION)); run(getConf(), input, output, measure, t1, t2, convergenceDelta, maxIterations); } return 0; } /** * Run the kmeans clustering job on an input dataset using the given the number of clusters k and iteration * parameters. All output data will be written to the output directory, which will be initially deleted if it exists. * The clustered points will reside in the path <output>/clustered-points. By default, the job expects a file * containing equal length space delimited data that resides in a directory named "testdata", and writes output to a * directory named "output". * * @param conf * the Configuration to use * @param input * the String denoting the input directory path * @param output * the String denoting the output directory path * @param measure * the DistanceMeasure to use * @param k * the number of clusters in Kmeans * @param convergenceDelta * the double convergence criteria for iterations * @param maxIterations * the int maximum number of iterations */ public static void run(Configuration conf, Path input, Path output, DistanceMeasure measure, int k, double convergenceDelta, int maxIterations) throws Exception { Path directoryContainingConvertedInput = new Path(output, DIRECTORY_CONTAINING_CONVERTED_INPUT); log.info("Preparing Input"); InputDriver.runJob(input, directoryContainingConvertedInput, "org.apache.mahout.math.RandomAccessSparseVector"); log.info("Running random seed to get initial clusters"); Path clusters = new Path(output, "random-seeds"); clusters = RandomSeedGenerator.buildRandom(conf, directoryContainingConvertedInput, clusters, k, measure); log.info("Running KMeans with k = {}", k); KMeansDriver.run(conf, directoryContainingConvertedInput, clusters, output, convergenceDelta, maxIterations, true, 0.0, false); // run ClusterDumper Path outGlob = new Path(output, "clusters-*-final"); Path clusteredPoints = new Path(output,"clusteredPoints"); log.info("Dumping out clusters from clusters: {} and clusteredPoints: {}", outGlob, clusteredPoints); ClusterDumper clusterDumper = new ClusterDumper(outGlob, clusteredPoints); //print the result clusterDumper.printClusters(null); } /** * Run the kmeans clustering job on an input dataset using the given distance measure, t1, t2 and iteration * parameters. All output data will be written to the output directory, which will be initially deleted if it exists. * The clustered points will reside in the path <output>/clustered-points. By default, the job expects the a file * containing synthetic_control.data as obtained from * http://archive.ics.uci.edu/ml/datasets/Synthetic+Control+Chart+Time+Series resides in a directory named "testdata", * and writes output to a directory named "output". * * @param conf * the Configuration to use * @param input * the String denoting the input directory path * @param output * the String denoting the output directory path * @param measure * the DistanceMeasure to use * @param t1 * the canopy T1 threshold * @param t2 * the canopy T2 threshold * @param convergenceDelta * the double convergence criteria for iterations * @param maxIterations * the int maximum number of iterations */ public static void run(Configuration conf, Path input, Path output, DistanceMeasure measure, double t1, double t2, double convergenceDelta, int maxIterations) throws Exception { Path directoryContainingConvertedInput = new Path(output, DIRECTORY_CONTAINING_CONVERTED_INPUT); log.info("Preparing Input"); InputDriver.runJob(input, directoryContainingConvertedInput, "org.apache.mahout.math.RandomAccessSparseVector"); log.info("Running Canopy to get initial clusters"); Path canopyOutput = new Path(output, "canopies"); CanopyDriver.run(new Configuration(), directoryContainingConvertedInput, canopyOutput, measure, t1, t2, false, 0.0, false); log.info("Running KMeans"); KMeansDriver.run(conf, directoryContainingConvertedInput, new Path(canopyOutput, Cluster.INITIAL_CLUSTERS_DIR + "-final"), output, convergenceDelta, maxIterations, true, 0.0, false); // run ClusterDumper ClusterDumper clusterDumper = new ClusterDumper(new Path(output, "clusters-*-final"), new Path(output, "clusteredPoints")); clusterDumper.printClusters(null); } }
上面的代码就是上一篇的example 例子，使用kmeans 实现聚集。
build.xml代码
<project name="mahout_test" default="jar"> <property name="root.dir" value="." /> <property name="src.dir" value="${root.dir}/src" /> <property name="lib.dir" value="${root.dir}/lib" /> <property name="build.dir" value="${root.dir}/build" /> <target name="clean" depends=""> <echo>root = ${root.dir}</echo> <delete dir="${build.dir}" /> <mkdir dir="${build.dir}" /> </target> <target name="build" depends="clean"> <javac fork="true" debug="true" srcdir="${src.dir}" destdir="${build.dir}"> <classpath> <fileset dir="${lib.dir}" includes="*.jar" /> </classpath> </javac> </target> <target name="jar" depends="build"> <mkdir dir="${build.dir}/lib" /> <!-- <copy file="${lib.dir}/mahout-core-0.9.jar" todir="${build.dir}/lib" /> <copy file="${lib.dir}/mahout-integration-0.9.jar" todir="${build.dir}/lib" /> <copy file="${lib.dir}/hadoop-core-1.2.1.jar" todir="${build.dir}/lib" /> --> <copy file="${lib.dir}/mahout-examples-0.9-job.jar" todir="${build.dir}/lib" /> <!-- <copy file="${lib.dir}/mahout-integration-0.9.jar" todir="${build.dir}/lib" /> --> <jar destfile="${root.dir}/mahout_test.jar" basedir="${build.dir}" > <manifest> <!-- <attribute name="Main-Class" value="chen/test/Job" /> --> </manifest> </jar> </target> </project>
编译命令：
ant -f build.xml
编译后，它会在${root.dir}下生成一个 mahout_test.jar 的文件。
编译程序依赖的jar包：mahout-core-0.9-job.jar、mahout-examples-0.9-job.jar、hadoop-core-1.2.1.jar
其中mahout-core-0.9.jar 包只是使用了org.slf4j.Logger、org.slf4j.LoggerFactory 类
你也可以依赖 hadoop lib 的 slf4j-api-1.4.3.jar 包来替换 mahout-core-0.9-job.jar 包。
制作Mahout 程序的关键在与在生成 jar 包时，要包含mahout-examples-0.9-job.jar 包。否则hadoop jar **.jar 运行是会出错。
<copy file="${lib.dir}/mahout-examples-0.9-job.jar" todir="${build.dir}/lib" />
mahout-examples-0.9-job.jar 包里面的类和 mahout-core-0.9-job.jar 包的类有很多是重叠的，这个实在太坑了。如果同时加载两个jar 包，它就报错，说找不到相应的类。
我被这个问题困扰了很久。
而且编译时，不要指定Main Class ，否则也会出错，原因我也没有细究，知道的同学可以留言。
运行命令：
bin/hadoop jar /mnt/hgfs/mnt/chenfool/mahout_test.jar chen.test.kmeans.TwoJob
运行的环境和上一篇的要求相似，也需要再 HDFS 的 /user/${user}/testdata 目录下存在向量文件。