在resources目录下添加hdfs-site.xml,core-site.xml,mapred-site.xml,yarn-site.xml,hbase-site.xml配置文件，直接从服务器复制就行了
Hdfs2Hbase代码编写步骤：
1、在pom.xml里添加依赖
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
2、主程序
import com.sun.jersey.core.util.StringIgnoreCaseKeyComparator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import java.io.IOException;
public class Hdfs2Hbase {
static String TABLE_NAME = "hdfs2hbase";
static String ZK_CONNECT_STR = "LianTian:2181";
public static class Hdfs2HbaseMapper extends Mapper
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
context.write(key, value);
}
}
public static class Hdfs2HBaseReducer extends TableReducer
@Override
protected void reduce(LongWritable key, Iterable
throws IOException, InterruptedException {
for (Text value : values) {
String[] words = value.toString().split(" ");
Put put = new Put(Bytes.toBytes(words[0]));
put.addColumn("info".getBytes(),"word".getBytes(),words[1].getBytes() );
context.write(NullWritable.get(), put);
}
}
}
public static void main(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", ZK_CONNECT_STR);
conf.set("mapred.jar", "D:\\大数据技术与应用专业\\HBase\\Code\\target\\Code-1.0-SNAPSHOT.jar");
System.setProperty("HADOOP_USER_NAME", "adolph");
Connection conn = ConnectionFactory.createConnection(conf);
Admin admin = conn.getAdmin();
TableName tableName = TableName.valueOf(TABLE_NAME);
if (!admin.tableExists(tableName)) {
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor hColumnDescriptor = new HColumnDescriptor("info".getBytes());
hColumnDescriptor.setValue("VERSIONS", "5");
desc.addFamily(hColumnDescriptor);
admin.createTable(desc);
}
Job job = Job.getInstance(conf);
job.setJarByClass(Hdfs2Hbase.class);
job.setMapperClass(Hdfs2HbaseMapper.class);
job.setReducerClass(Hdfs2HBaseReducer.class);
TableMapReduceUtil.initTableReducerJob(TABLE_NAME, Hdfs2HBaseReducer.class, job);
// hdfs集群上词频统计后的文件
FileInputFormat.setInputPaths(job, new Path("hdfs://LianTian:9000/1.txt"));
// map阶段的输出类型
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(Text.class);
// 最终的输出类型
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Put.class);
// 提交job
boolean statue = job.waitForCompletion(true);
System.exit(statue ? 0 : 1);
}
}
3、在idea编写hdfs2hbase代码运行出现以下错误
4、解决方法：在mapred-site.xml加入如下内容
精彩链接
您阅读本篇文章共花了：
发表评论