1, Experimental purpose

(1) Understand the role of HDFS in Hadoop architecture;

(2) Proficient in using HDFS to operate common Shell commands;

(3) Familiar with Java API s commonly used in HDFS operation.

2, Experimental platform

• Operating system: Linux (CentOS recommended);
• Hadoop version: 3.2.2;
• HBase version: 2.3.6;
• JDK version: 1.7 or above;
• Java IDE: IDEA

3, Experimental steps

(1) Program to realize the following specified functions, and complete the same task with HBase Shell command provided by Hadoop:

1. List the relevant information of all HBase tables, such as table name;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import java.io.IOException;
import java.io.IOException;
public class test2 {
    public static Configuration configuration;
    public static Connection connection;
    public static Admin admin;
    public static void init(){
        configuration = HBaseConfiguration.create();
        configuration.set("Hbase.rootdir","hdfs://localhost:9000/hbase");
        try{
            connection = ConnectionFactory.createConnection(configuration);
            admin = connection.getAdmin();
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void close(){
        try {
            if(admin != null) {
                admin.close();
            }
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void getData()throws  IOException{
        TableName Htable[] = admin.listTableNames();
        for (TableName name:Htable){
            System.out.println(name.getNameAsString());
            Table table = connection.getTable(name);
            TableDescriptor HTableDes = table.getDescriptor();
        }
        System.out.println(admin.listTableDescriptors());
    }
    public static void main(String[] args)throws IOException{
        init();
        getData();
        close();
    }
}

2 print out all recorded data of the specified table at the terminal;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
public class test2 {
    public static Configuration configuration;
    public static Connection connection;
    public static Admin admin;
    public static void init(){
        configuration = HBaseConfiguration.create();
        configuration.set("Hbase.rootdir","hdfs://localhost:9000/hbase");
        try{
            connection = ConnectionFactory.createConnection(configuration);
            admin = connection.getAdmin();
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void close(){
        try {
            if(admin != null) {
                admin.close();
            }
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void printTableData(String tableName) throws IOException{
        Table table = connection.getTable(TableName.valueOf(tableName));
        Scan scan = new Scan();
        scan.getAllowPartialResults();
        ResultScanner resultScanner = table.getScanner(scan);
        for(Result result:resultScanner){
            Cell[] cells = result.rawCells();
            for (Cell cell : cells) {
                System.out.print("Row key:" + Bytes.toString(CellUtil.cloneRow(cell)));
                System.out.print("\t Column family" + Bytes.toString(CellUtil.cloneFamily(cell)));
                System.out.print("\t column:" + Bytes.toString(CellUtil.cloneQualifier(cell)));
                System.out.println("\t value:" + Bytes.toString(CellUtil.cloneValue(cell)));
                System.out.println("\t time stamp:" + cell.getTimestamp());
            }
        }
        table.close();
    }
    public static void main(String[] args)throws IOException{
        init();
        printTableData("student");
        close();
    }
}

3 add and delete the specified column family or column to the created table;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
public class test2 {
    public static Configuration configuration;
    public static Connection connection;
    public static Admin admin;
    public static void init(){
        configuration = HBaseConfiguration.create();
        configuration.set("Hbase.rootdir","hdfs://localhost:9000/hbase");
        try{
            connection = ConnectionFactory.createConnection(configuration);
            admin = connection.getAdmin();
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void close(){
        try {
            if(admin != null) {
                admin.close();
            }
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void modifyTable(String tableName, String RowKey, String columnFamily, String column, String value) throws IOException{
        Table table = connection.getTable(TableName.valueOf(tableName));
        Put put = new Put(RowKey.getBytes());
        put.addColumn(columnFamily.getBytes(),column.getBytes(),value.getBytes());
        table.put(put);
        table.close();
    }
    public static void modifyTable(String tableName, String RowKey, String columnFamily, String column) throws IOException{
        Table table = connection.getTable(TableName.valueOf(tableName));
        Delete delete = new Delete(Bytes.toBytes(RowKey));
        delete.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column));
        //Deletes the specified column family
        //delete.addFamily(Bytes.toBytes(columnFamily));
        table.delete(delete);
        table.close();
    }
    public static void main(String[] args)throws IOException{
        init();
        modifyTable("student","zhangsan","score","English","80");//Add specified column
        modifyTable("student","zhangsan","score","English");//Deletes the specified column
        close();
    }
}

4 clear all record data of the specified table;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import java.io.IOException;
public class test2 {
    public static Configuration configuration;
    public static Connection connection;
    public static Admin admin;
    public static void init(){
        configuration = HBaseConfiguration.create();
        configuration.set("Hbase.rootdir","hdfs://localhost:9000/hbase");
        try{
            connection = ConnectionFactory.createConnection(configuration);
            admin = connection.getAdmin();
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void close(){
        try {
            if(admin != null) {
                admin.close();
            }
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void empty(String tableName) throws IOException{
        Table table = connection.getTable(TableName.valueOf(tableName));
        Scan scan = new Scan();
        scan.getAllowPartialResults();
        ResultScanner resultScanner = table.getScanner(scan);
        for(Result result:resultScanner){
            Cell[] cells = result.rawCells();
            for (Cell cell : cells) {
                Delete delete = new Delete(CellUtil.cloneRow(cell));
                table.delete(delete);
            }
        }
        System.out.println("Delete complete");
    }
    public static void main(String[] args)throws IOException{
        init();
        empty("student");
        close();
    }
}

5 count the number of rows in the table.

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.client.coprocessor.*;
import java.io.IOException;
public class test2 {
    public static Configuration configuration;
    public static Connection connection;
    public static Admin admin;
    public static void init(){
        configuration = HBaseConfiguration.create();
        configuration.set("Hbase.rootdir","hdfs://localhost:9000/hbase");
        try{
            connection = ConnectionFactory.createConnection(configuration);
            admin = connection.getAdmin();
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void close(){
        try {
            if(admin != null) {
                admin.close();
            }
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void rowCount(String tableName){
        try {
            TableName name = TableName.valueOf(tableName);
            if(admin.tableExists(name)) {
                Scan scan = new Scan();
                AggregationClient aggregationClient = new AggregationClient(configuration);//Starting the global aggregation scan is faster. In HBase site, XML is modified
                System.out.println("RowCount: " + aggregationClient.rowCount(name, new LongColumnInterpreter(), scan));
            }else System.out.println("table not exist");
        }catch (Throwable e){
            e.printStackTrace();
        }
    }
    public static void main(String[] args)throws IOException{
        init();
        rowCount("player");
        close();
    }
}

(2) HBase database operation

(1) createTable(String tableName, String[] fields) creates a table. The parameter tableName is the name of the table, and the string array fields is the array that stores the names of each field of the record. When a table named tableName already exists in HBase, delete the original table first, and then create a new table.

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
public class test2 {
    public static Configuration configuration;
    public static Connection connection;
    public static Admin admin;
    public static void init(){
        configuration = HBaseConfiguration.create();
        configuration.set("Hbase.rootdir","hdfs://localhost:9000/hbase");
        try{
            connection = ConnectionFactory.createConnection(configuration);
            admin = connection.getAdmin();
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void close(){
        try {
            if(admin != null) {
                admin.close();
            }
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void createTable(String myTableName, String[] fields) throws  IOException{
        TableName tableName= TableName.valueOf(myTableName);
        if(admin.tableExists(tableName)) {
            System.out.println("table exist");
            admin.disableTable(tableName);
            admin.deleteTable(tableName);
        }
        TableDescriptorBuilder tableDescriptor = TableDescriptorBuilder.newBuilder(tableName);
        for(String str:fields){
            ColumnFamilyDescriptor family = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(str)).build();
            tableDescriptor.setColumnFamily(family);
        }
        admin.createTable(tableDescriptor.build());
        System.out.println("table created");

    }
    public static void main(String[] args)throws IOException{
        init();
        createTable("table2",new String[] {"score"});
        close();
    }
}

(2) addRecord(String tableName, String row, String[] fields, String[] values) adds corresponding data values to the cells specified by table tableName, row row (represented by S_Name) and string array fields. If there is a corresponding column qualifier under the corresponding column family for each element in fields, it is represented by "columnFamily:column". For example, when adding grades to the "Math", "computer science" and "English" columns at the same time, the string array fields is {Score:Math "," Score:Computer Science "," Score:English "}, and the array values stores the grades of the three courses.

(3) scanColumn(String tableName, String column) browses the data of a column in the tableName table. If the column data does not exist in a row record, null will be returned. When the parameter column is a column family name, if there are several column qualifiers below, the data of the column represented by each column qualifier will be listed; when the parameter column is a column specific name (for example "Score:Math"), you only need to list the data of this column.

(4) modifyData(String tableName, String row, String column) modifies the data of the cells specified in the table tableName, row row (which can be represented by student name S_Name) and column column.

(5) deleteRow(String tableName, String row) deletes the record of the row specified by row in table tableName.

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
public class test2 {
    public static Configuration configuration;
    public static Connection connection;
    public static Admin admin;
    public static void init(){
        configuration = HBaseConfiguration.create();
        configuration.set("Hbase.rootdir","hdfs://localhost:9000/hbase");
        try{
            connection = ConnectionFactory.createConnection(configuration);
            admin = connection.getAdmin();
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void close(){
        try {
            if(admin != null) {
                admin.close();
            }
        }catch (IOException e){
            e.printStackTrace();
        }
    }
    public static void deleteRow(String tableName,String row) throws IOException{
        Table table = connection.getTable(TableName.valueOf(tableName));
        Delete delete = new Delete(Bytes.toBytes(row));
        table.delete(delete);
        table.close();
    }
    public static void main(String[] args)throws IOException{
        init();
        deleteRow("student", "score");
        close();
    }
}