使用okhttp工具集来开发：（如果文件已经存在会报错）
package com.etl;import java.io.IOException;import org.apache.commons.lang3.StringUtils;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import okhttp3.Call;import okhttp3.Callback;import okhttp3.OkHttpClient;import okhttp3.Request;import okhttp3.Response;public class LinkHttp { ???????private ?static Configuration conf = null; ???private static String fsName = "fs.defaultFS"; ???private static String fsURI = null; ???public static void main(String[] args) throws Exception { ???????????????String name = args[0]; ???????String uri = ?args[1]; ???????String url = args[2]; ???????String targetFile = args[3]; ?//文件全路径 ???????????????//初始化 ???????init(name, uri); ???????OkHttpClient client = new OkHttpClient(); ???????final Request request = new Request.Builder().url(url).get().build(); ???????Call call = client.newCall(request); ???????call.enqueue(new Callback() { ???????????@Override ???????????public void onFailure(Call call, IOException e) { ???????????????System.out.println("Fail"); ???????????} ???????????@Override ???????????public void onResponse(Call call, Response response) throws IOException { ???????????????FileSystem fs = null; ???????????????try { ???????????????????????Path dstPath = new Path(targetFile); ???????????????????????fs = FileSystem.get(conf); ???????????????????????FSDataOutputStream outputStream = fs.create(dstPath); ???????????????????????????if(response.isSuccessful()) { ???????????// ???????????????????System.out.println(response.body().string()); ???????????????????????????????outputStream.write(response.body().bytes()); ???????????????????????????????????????outputStream.close(); ???????????????????????????????????????System.out.println("create file " + targetFile + " success!"); ???????????????????????????????????????//fs.close(); ???????????????????????} ???????????????}catch (Exception e){ ???????????????????e.printStackTrace(); ???????????????}finally { ???????????????????fs.close(); ???????????????} ???????????????System.out.println("run writeHdfs end"); ???????????????//关闭 ???????????????if(response.body()!=null) { ???????????????????response.body().close(); ???????????????} ???????????} ???????}); ???????????} ???private static void init(String name, String uri) { ???????????if(StringUtils.isNotBlank(fsName)){ ???????????????fsName = name; ???????????} ????????????fsURI ?= uri; ????????????conf = new Configuration(); ????????????conf.set(fsName, fsURI); ???}}配置启动脚本如下：
#!/bin/shname=fs.defaultFS ???????#固定不变uri=dwpro-name1:8020 ????#hdfs文件系统地址url=http://www.cnblogs.com/30go/ ??#待保存的http地址targetPath=/tmp/test/king.txt ??# 目标的文件名java -Djava.ext.dirs=lib com.etl.LinkHttp ${name} ${uri} ${url} ${targetPath} ?>> test.log 2>&1 &简单的Http请求数据保存到Hdfs
原文地址：https://www.cnblogs.com/30go/p/9762527.html