java操作hdfs

  1 package hdfs;
  2 
  3 import java.io.FileNotFoundException;
  4 import java.io.IOException;
  5 import java.net.URI;
  6 
  7 import org.apache.hadoop.conf.Configuration;
  8 import org.apache.hadoop.fs.FileStatus;
  9 import org.apache.hadoop.fs.FileSystem;
 10 import org.apache.hadoop.fs.LocatedFileStatus;
 11 import org.apache.hadoop.fs.Path;
 12 import org.apache.hadoop.fs.RemoteIterator;
 13 import org.junit.Before;
 14 import org.junit.Test;
 15 
 16 public class HdfsClientEasy {
 17 
 18     private FileSystem fs = null;
 19 
 20     @Before
 21     public void getFs() throws Exception {
 22 
 23         // 拿到一个配置参数的封装对象，构造函数中就会对classpath下的xxx-site.xml文件进行解析
 24         // 真实的项目工程中就应该把xxx-site.xml文件加入到工程中来
 25         Configuration conf = new Configuration();
 26         // to set a parameter, figure out the filesystem is hdfs
 27         conf.set("fs.defaultFS", "hdfs://localhost:8000/");
 28         conf.set("dfs.replication", "1");
 29 
 30         // 获取到一个具体文件系统的客户端实例对象，产生的实例究竟是哪一种文件系统的客户端，是根据conf中的相关参数来决定
 31 //        fs = FileSystem.get(conf);
 32         
 33         //这种获取fs的方法可以指定访问hdfs的客户端身份
 34         fs = FileSystem.get(new URI("hdfs://localhost:8000/"), conf, "wochu");
 35 
 36     }
 37 
 38     /**
 39      * 上传文件
 40      * 
 41      * @throws IllegalArgumentException
 42      * @throws IOException
 43      */
 44     @Test
 45     public void testUpload() throws IllegalArgumentException, IOException {
 46         fs.copyFromLocalFile(new Path("/Users/wochu/Desktop/tool/hdfs/test.txt"), new Path("/xue"));
 47 
 48     }
 49 
 50     /**
 51      * 删除文件
 52      * 
 53      * @throws IllegalArgumentException
 54      * @throws IOException
 55      */
 56     @Test
 57     public void testRmfile() throws IllegalArgumentException, IOException {
 58 
 59         boolean res = fs.delete(new Path("/xue"), true);
 60 
 61         System.out.println(res ? "delete is successfully :)"
 62                 : "it is failed :(");
 63 
 64     }
 65 
 66     /**
 67      * 创建文件夹
 68      * 
 69      * @throws IllegalArgumentException
 70      * @throws IOException
 71      */
 72     @Test
 73     public void testMkdir() throws IllegalArgumentException, IOException {
 74         fs.mkdirs(new Path("/xue"));
 75 
 76     }
 77 
 78     /**
 79      * 重命名文件
 80      * 
 81      * @throws IllegalArgumentException
 82      * @throws IOException
 83      */
 84     @Test
 85     public void testRename() throws IllegalArgumentException, IOException {
 86 
 87         fs.rename(new Path("/xue/test.txt"), new Path("/xue/rename.txt"));
 88 
 89     }
 90 
 91     /**
 92      * 列出目录下的文件信息
 93      * 
 94      * @throws FileNotFoundException
 95      * @throws IllegalArgumentException
 96      * @throws IOException
 97      */
 98     @Test
 99     public void testListFiles() throws FileNotFoundException,
100             IllegalArgumentException, IOException {
101 
102         // 递归列出文件
103         RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(
104                 new Path("/"), true);
105 
106         while (listFiles.hasNext()) {
107 
108             LocatedFileStatus file = listFiles.next();
109 
110             System.out.println(file.getPath().getName());
111 
112         }
113 
114         System.out.println("--------------------------------------------");
115 
116         // 列出文件及文件夹
117         FileStatus[] status = fs.listStatus(new Path("/"));
118         for (FileStatus file : status) {
119 
120             System.out.println(file.getPath().getName() + "   "
121                     + (file.isDirectory() ? "d" : "f"));
122 
123         }
124 
125     }
126 
127     /**
128      * 从hdfs中下载数据到本地
129      * 
130      * @throws IllegalArgumentException
131      * @throws IOException
132      */
133     @Test
134     public void testDownload() throws IllegalArgumentException, IOException {
135 
136         fs.copyToLocalFile(new Path("/xue/rename.txt"), new Path("/Users/wochu/Desktop/tool/hdfs/name.txt"));
137 
138     }
139 
140 }