Overview

Learn how to interact with Hadoop 2.6.0 using the Java client library. hdfs_skim is a String type variable in the format hdfs://{active_namenode_url}:{port}.

Hadoop Client Installation
hadoop-client

Hadoop Client Installation

Find the Maven repository matching your Hadoop version from hadoop-client Maven Repository.

pom.xml

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.6.0</version>
        </dependency>
    </dependencies>

hadoop-client

create directory

Add a folder named test under the /tmp/ directory in hdfs.

create_directory

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.Configuration;

    public static void createDirectory() throws IOException {
        // create directory
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", hdfs_skim);
        FileSystem fileSystem = FileSystem.get(configuration);
        String directoryName = "/tmp/test";
        Path path = new Path(directoryName);
        fileSystem.mkdirs(path);
    }

new_directory_check

 hdfs dfs -ls /tmp
 drwxr-xr-x   - user   supergroup          0 2022-10-03 19:04 /tmp/test

create file

Add a text file named read_write_hdfs_example.txt under the /tmp/test directory in hdfs.

add

import org.apache.hadoop.fs.FSDataOutputStream;
import java.io.BufferedWriter;

    public static void writeFileToHDFS() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", hdfs_skim);
        FileSystem fileSystem = FileSystem.get(configuration);
        //Create a path
        String fileName = "read_write_hdfs_example.txt";
        Path hdfsWritePath = new Path("/tmp/test/" + fileName);
        FSDataOutputStream fsDataOutputStream = fileSystem.create(hdfsWritePath,true);

        BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream,StandardCharsets.UTF_8));
        bufferedWriter.write("Java API to write data in HDFS");
        bufferedWriter.newLine();
        bufferedWriter.close();
        fileSystem.close();
    }

new_file_check

hdfs dfs -ls /tmp/test
Found 1 items
-rw-r--r--   3 user supergroup         31 2022-10-03 19:40 /tmp/test/read_write_hdfs_example.txt

edit file

Add a text line to the /tmp/test/read_write_hdfs_example.txt file in hdfs.

edit

import org.apache.hadoop.fs.FSDataOutputStream;
import java.io.BufferedWriter;

    public static void writeFileToHDFS() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", hdfs_skim);
        FileSystem fileSystem = FileSystem.get(configuration);
        //Create a path
        String fileName = "read_write_hdfs_example.txt";
        Path hdfsWritePath = new Path("/tmp/test/" + fileName);
        FSDataOutputStream fsDataOutputStream = fileSystem.create(hdfsWritePath,true);

        BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream,StandardCharsets.UTF_8));
        bufferedWriter.write("Java API to write data in HDFS");
        bufferedWriter.newLine();
        bufferedWriter.close();
        fileSystem.close();
    }

edit_file_check

hdfs dfs -cat /tmp/test/read_write_hdfs_example.txt
Java API to write data in HDFS
Java API to append data in HDFS file

read file

Read the /tmp/test/read_write_hdfs_example.txt file from hdfs.

edit

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import java.io.*;

    public static void readFileFromHDFS() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", hdfs_skim);
        FileSystem fileSystem = FileSystem.get(configuration);
        //Create a path
        String fileName = "read_write_hdfs_example.txt";
        Path hdfsReadPath = new Path("/tmp/test/" + fileName);
        //Init input stream
        FSDataInputStream inputStream = fileSystem.open(hdfsReadPath);
        //Classical input stream usage
//        String out= IOUtils.toString(inputStream, "UTF-8");
//        System.out.println(out);

        BufferedReader bufferedReader = new BufferedReader(
                new InputStreamReader(inputStream, StandardCharsets.UTF_8));

        String line = null;
        while ((line=bufferedReader.readLine())!=null){
            System.out.println(line);
        }

        inputStream.close();
        fileSystem.close();
    }

output

Java API to write data in HDFS
Java API to append data in HDFS file

Process finished with exit code 0