hdfs：
Hadoop分布式文件系统(HDFS)是指被设计成适合运行在通用硬件(commodity hardware)上的分布式文件系统（Distributed File System）。它和现有的分布式文件系统有很多共同点。但同时，它和其他的分布式文件系统的区别也是很明显的。HDFS是一个高度容错性的系统，适合部署在廉价的机器上。HDFS能提供高吞吐量的数据访问，非常适合大规模数据集上的应用。HDFS放宽了一部分POSIX约束，来实现流式读取文件系统数据的目的。HDFS在最开始是作为Apache Nutch搜索引擎项目的基础架构而开发的。HDFS是Apache Hadoop Core项目的一部分。
Java操作工具类：
import lombok.RequiredArgsConstructor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.stereotype.Component;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
/**
* hdfs工具
*/
@Component
@RequiredArgsConstructor
public class HdfsUtils {
//配置链接虚拟机的IP
public static final String HDFS_PATH = "hdfs://192.168.10.150:8020";
//hdfs文件系统
FileSystem fileSystem = null;
//获取环境对象
Configuration configuration = null;
/**
* 新建目录
*
* @throws IOException
*/
@Test
public void mkdir() throws IOException {
fileSystem.mkdirs(new Path("/user/filestore/history/2021/10/11/"));
}
//hadoop fs -ls /hdfsapi/test
/**
* 创建文件
*
* @throws IOException
*/
@Test
public void create() throws IOException {
//创建文件
FSDataOutputStream outputStream = fileSystem.create(new Path("/user/filestore/test/l.txt"));
outputStream.write("hello hadoop".getBytes());
outputStream.flush();
outputStream.close();
}
/**
* 打印
*
* @throws IOException
*/
@Test
public void cat() throws IOException {
FSDataInputStream inputStream = fileSystem.open(new Path("/user/filestore/test/b.txt"));
IOUtils.copyBytes(inputStream, System.out, 1024);
inputStream.close();
}
/**
* 重命名
*
* @throws IOException
*/
@Test
public void rename() throws IOException {
Path oldPath = new Path("/user/filestore/current/22/76/33/t_file_history.sql");
Path newPath = new Path("/user/filestore/history/2021/10/11/t_file_history.sql");
Assert.assertTrue(fileSystem.rename(oldPath, newPath));
}
/**
* 上传本地文件到hdfs
*
* @throws Exception
*/
@Test
public void copyFromLocalFile() throws Exception {
Path oldPath = new Path("D:\file/aa/20210908_0600.png");
Path newPath = new Path("/user/filestore/filehistory/2021");
fileSystem.copyFromLocalFile(oldPath, newPath);
}
/**
* 上传本地文件到hdfs 带进度条
*
* @throws Exception
*/
@Test
public void copyFromLocalFileWithProgress() throws Exception {
// Path oldPath = new Path("E:\ideaIU-2018.1.exe");
// Path newPath = new Path("/user/filestore/test");
// fileSystem.copyFromLocalFile(oldPath,newPath);
InputStream in = new BufferedInputStream(
new FileInputStream(
new File("E:\文档.txt")));
FSDataOutputStream outputStream = fileSystem.create(new Path("/user/filestore/test/文档.txt"), new Progressable() {
@Override
public void progress() {
System.out.print(".");
//带进度提醒信息
}
});
IOUtils.copyBytes(in, outputStream, 4096);
}
/**
* 下载文件到本地
*
* @throws Exception
*/
@Test
public void copyToLocalFile() throws Exception {
// 获取输入流
InputStream in = fileSystem.open(new Path("/user/filestore/filehistory/2021/20210908_0600.png"));
// 获取输出流
OutputStream outputStream = new FileOutputStream(new File("D:\file/a.png"));
IOUtils.copyBytes(in, outputStream, configuration);
in.close();
outputStream.close();
}
/**
* 列出所有的文件
*
* @throws URISyntaxException
* @throws IOException
*/
@Test
public void listFiles() throws Exception {
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/user"));
for (FileStatus fileStatus : fileStatuses) {
String isDir = fileStatus.isDirectory() ? "文件夹" : "文件";
short relication = fileStatus.getReplication();
long len = fileStatus.getLen();
String path = fileStatus.getPath().toString();
System.out.println(isDir + ":" + relication + ":" + len + ":" + path);
}
}
@Test
public void upset() throws URISyntaxException, IOException {
//上传文件,路径大家记得改一下
String file = "D:/hadoop/output/test.txt";
InputStream inputStream = new FileInputStream(new File(file));
FSDataOutputStream outputStream = fileSystem.create(new Path("/user/filestore/park/aaa.txt"));
IOUtils.copyBytes(inputStream, outputStream, configuration);
// fileSystem.copyFromLocalFile();底层是调用了IOUtils.copyBytes()
}
@Test
public void download() throws URISyntaxException, IOException {
// 获取输入流
InputStream in = fileSystem.open(new Path("/park/2.txt"));
// 获取输出流
String file = "D:/hadoop/output/test.txt";
OutputStream outputStream = new FileOutputStream(new File(file));
IOUtils.copyBytes(in, outputStream, configuration);
in.close();
outputStream.close();
}
/**
* 删除hdfs文件
*
* @throws Exception
*/
@Test
public void deleteFile() throws Exception {
Path path = new Path("/user/filestore/history");
fileSystem.delete(path, true);
}
@Test
public void demo1() throws URISyntaxException, IOException {
configuration = new Configuration();
fileSystem = (FileSystem) FileSystem.get(new URI(HDFS_PATH), configuration);
// 1、在hdfs创建目录teacher。
// 2、在teacher目录下上传文件score.txt。
String file = "D:/hadoop/score.txt";
InputStream inputStream = new FileInputStream(new File(file));
OutputStream outputStream = fileSystem.create(new Path("/hdfs/teacher/score.txt"));
IOUtils.copyBytes(inputStream, outputStream, configuration);
// 3、在hdfs创建目录student，并在student目录下创建新目录Tom、LiMing、Jerry.
fileSystem.mkdirs(new Path("/hdfs/student/Tom"));
fileSystem.mkdirs(new Path("/hdfs/student/LiMing"));
fileSystem.mkdirs(new Path("/hdfs/student/Jerry"));
// 4、在Tom目录下上传information.txt,同时上传到LiMing、Jerry目录下。
file = "D:/hadoop/information.txt";
inputStream = new FileInputStream(new File(file));
outputStream = fileSystem.create(new Path("/hdfs/student/Tom/information.txt"));
IOUtils.copyBytes(inputStream, outputStream, configuration);
// file = "D:/hadoop/information.txt";
inputStream = new FileInputStream(new File(file));
outputStream = fileSystem.create(new Path("/hdfs/student/LiMing/information.txt"));
IOUtils.copyBytes(inputStream, outputStream, configuration);
// file = "D:/hadoop/information.txt";
inputStream = new FileInputStream(new File(file));
outputStream = fileSystem.create(new Path("/hdfs/student/Jerry/information.txt"));
IOUtils.copyBytes(inputStream, outputStream, configuration);
// 5、将student重命名为MyStudent。
fileSystem.rename(new Path("/hdfs/student"), new Path("/hdfs/MyStudent"));
// 6、将Tom下的information.txt下载到E:/tom目录中
file = "E:/tom";
inputStream = fileSystem.open(new Path("/hdfs/MyStudent/Tom/information.txt"));
outputStream = new FileOutputStream(new File(file));
IOUtils.copyBytes(inputStream, outputStream, configuration);
// 7、将teacher下的score.txt也下载到此目录
inputStream = fileSystem.open(new Path("/hdfs/teacher/score.txt"));
outputStream = new FileOutputStream(new File(file));
IOUtils.copyBytes(inputStream, outputStream, configuration);
// 8、删除hdfs中的Tom、LiMing目录
fileSystem.delete(new Path("/hdfs/Tom"), true);
fileSystem.delete(new Path("/hdfs/LiMing"), true);
inputStream.close();
outputStream.close();
}
@Before
public void setUp() throws Exception {
configuration = new Configuration();
fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "root");
System.out.println("HDFSApp setUp");
}
@After
public void tearDown() throws Exception {
configuration = null;
fileSystem = null;
System.out.println("HDFSApp tearDown");
}
}