C#爬虫基础之HttpClient获取HTTP请求与响应
作者：springsnow
这篇文章介绍了C#使用HttpClient获取HTTP请求与响应的方法，文中通过示例代码介绍的非常详细。对大家的学习或工作具有一定的参考借鉴价值，需要的朋友可以参考下
一、概述
Net4.5以上的提供基本类，用于发送 HTTP 请求和接收来自通过 URI 确认的资源的 HTTP 响应。
HttpClient是一个高级 API，用于包装其运行的每个平台上可用的较低级别功能。
// HttpClient is intended to be instantiated once per application, rather than per-use. See Remarks.
static readonly HttpClient client = new HttpClient();
static async Task Main()
{
// Call asynchronous network methods in a try/catch block to handle exceptions.
try
{
HttpResponseMessage response = await client.GetAsync("http://www.contoso.com/");
response.EnsureSuccessStatusCode();
string responseBody = await response.Content.ReadAsStringAsync();
// Above three lines can be replaced with new helper method below
// string responseBody = await client.GetStringAsync(uri);
Console.WriteLine(responseBody);
}
catch(HttpRequestException e)
{
Console.WriteLine("\nException Caught!");
Console.WriteLine("Message :{0} ",e.Message);
}
}二、HttpClient的使用
1.使用HttpClient调用Oauth的授权接口获取access_token
1）OAuth使用的密码式
2）获取到access_token后才进行下一步
2.带着access_token调用接口
1）hearder上添加bearer方式的access_token
2）调用接口确保成功获取到返回的结果
try
{
string host = ConfigurationManager.AppSettings["api_host"];
string username = ConfigurationManager.AppSettings["api_username"];
string password = ConfigurationManager.AppSettings["api_password"];
HttpClient httpClient = new HttpClient();
// 设置请求头信息
httpClient.DefaultRequestHeaders.Add("Host", host);
httpClient.DefaultRequestHeaders.Add("Method", "Post");
httpClient.DefaultRequestHeaders.Add("KeepAlive", "false"); // HTTP KeepAlive设为false，防止HTTP连接保持
httpClient.DefaultRequestHeaders.Add("UserAgent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
//获取token
var tokenResponse = httpClient.PostAsync("http://" + host + "/token", new FormUrlEncodedContent(new Dictionary<string, string> {
{"grant_type","password"},
{"username", username},
{"password", password}
}));
tokenResponse.Wait();
tokenResponse.Result.EnsureSuccessStatusCode();
var tokenRes = tokenResponse.Result.Content.ReadAsStringAsync();
tokenRes.Wait();
var token = Newtonsoft.Json.Linq.JObject.Parse(tokenRes.Result);
var access_token = token["access_token"].ToString();
// 调用接口发起POST请求
var authenticationHeaderValue = new AuthenticationHeaderValue("bearer", access_token);
httpClient.DefaultRequestHeaders.Authorization = authenticationHeaderValue;
var content = new StringContent(parameter);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
var response = httpClient.PostAsync("http://" + host + "/" + api_address, content);
response.Wait();
response.Result.EnsureSuccessStatusCode();
var res = response.Result.Content.ReadAsStringAsync();
res.Wait();return Newtonsoft.Json.JsonConvert.DeserializeObject(res.Result);
}
catch (Exception ex)
{
return ResultEx.Init(ex.Message);
}HttpClient 获取图片并保存到本机
class Program
{
static void Main()
{
//图片路径：https://img.infinitynewtab.com/wallpaper/1.jpg
string imgSourceURL = "https://img.infinitynewtab.com/wallpaper/";
DownloadImags(imgSourceURL).Wait();
}
private static async Task DownloadImags(string url)
{
var client = new HttpClient();
System.IO.FileStream fs;
int a = 1;
//文件名：序号+.jpg。可指定范围，以下是获取100.jpg~500.jpg.
for (int i = 100; i <= 500; i++)
{
var uri = new Uri(Uri.EscapeUriString(url+i.ToString()+".jpg"));
byte[] urlContents = await client.GetByteArrayAsync(uri);
fs = new System.IO.FileStream(AppDomain.CurrentDomain.BaseDirectory + "\\images\\"+ i.ToString() + ".jpg",System.IO.FileMode.CreateNew);
fs.Write(urlContents, 0, urlContents.Length);
Console.WriteLine(a++);
}
}
}以下为封装的类库
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Serialization;
public class HttpClientHelpClass
{
///
/// get请求
///
///
///
public static string GetResponse(string url, out string statusCode)
{
if (url.StartsWith("https"))
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Accept.Add( new MediaTypeWithQualityHeaderValue("application/json"));
HttpResponseMessage response = httpClient.GetAsync(url).Result;
statusCode = response.StatusCode.ToString();
if (response.IsSuccessStatusCode)
{
string result = response.Content.ReadAsStringAsync().Result;
return result;
}
return null;
}
public static string RestfulGet(string url)
{
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
// Get response
using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
{
// Get the response stream
StreamReader reader = new StreamReader(response.GetResponseStream());
// Console application output
return reader.ReadToEnd();
}
}
public static T GetResponse(string url)
where T : class, new()
{
if (url.StartsWith("https"))
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Accept.Add( new MediaTypeWithQualityHeaderValue("application/json"));
HttpResponseMessage response = httpClient.GetAsync(url).Result;
T result = default(T);
if (response.IsSuccessStatusCode)
{
Task<string> t = response.Content.ReadAsStringAsync();
string s = t.Result;
result = JsonConvert.DeserializeObject(s);
}
return result;
}
///
/// post请求
///
///
/// post数据
///
public static string PostResponse(string url, string postData, out string statusCode)
{
if (url.StartsWith("https"))
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
HttpContent httpContent = new StringContent(postData);
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
httpContent.Headers.ContentType.CharSet = "utf-8";
HttpClient httpClient = new HttpClient();
//httpClient..setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8");
HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result;
statusCode = response.StatusCode.ToString();
if (response.IsSuccessStatusCode)
{
string result = response.Content.ReadAsStringAsync().Result;
return result;
}
return null;
}
///
/// 发起post请求
///
///
/// url
/// post数据
///
public static T PostResponse(string url, string postData)
where T : class, new()
{
if (url.StartsWith("https"))
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
HttpContent httpContent = new StringContent(postData);
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
HttpClient httpClient = new HttpClient();
T result = default(T);
HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result;
if (response.IsSuccessStatusCode)
{
Task<string> t = response.Content.ReadAsStringAsync();
string s = t.Result;
result = JsonConvert.DeserializeObject(s);
}
return result;
}
///
/// 反序列化Xml
///
///
///
///
public static T XmlDeserialize(string xmlString)
where T : class, new()
{
try
{
XmlSerializer ser = new XmlSerializer(typeof(T));
using (StringReader reader = new StringReader(xmlString))
{
return (T)ser.Deserialize(reader);
}
}
catch (Exception ex)
{
throw new Exception("XmlDeserialize发生异常：xmlString:" + xmlString + "异常信息：" + ex.Message);
}
}
public static string PostResponse(string url, string postData, string token, string appId, string serviceURL, out string statusCode)
{
if (url.StartsWith("https"))
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
HttpContent httpContent = new StringContent(postData);
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
httpContent.Headers.ContentType.CharSet = "utf-8";
httpContent.Headers.Add("token", token);
httpContent.Headers.Add("appId", appId);
httpContent.Headers.Add("serviceURL", serviceURL);
HttpClient httpClient = new HttpClient();
//httpClient..setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8");
HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result;
statusCode = response.StatusCode.ToString();
if (response.IsSuccessStatusCode)
{
string result = response.Content.ReadAsStringAsync().Result;
return result;
}
return null;
}
///
/// 修改API
///
///
///
///
public static string KongPatchResponse(string url, string postData)
{
var httpWebRequest = (HttpWebRequest)WebRequest.Create(url);
httpWebRequest.ContentType = "application/x-www-form-urlencoded";
httpWebRequest.Method = "PATCH";
byte[] btBodys = Encoding.UTF8.GetBytes(postData);
httpWebRequest.ContentLength = btBodys.Length;
httpWebRequest.GetRequestStream().Write(btBodys, 0, btBodys.Length);
HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
var streamReader = new StreamReader(httpWebResponse.GetResponseStream());
string responseContent = streamReader.ReadToEnd();
httpWebResponse.Close();
streamReader.Close();
httpWebRequest.Abort();
httpWebResponse.Close();
return responseContent;
}
///
/// 创建API
///
///
///
///
public static string KongAddResponse(string url, string postData)
{
if (url.StartsWith("https"))
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
HttpContent httpContent = new StringContent(postData);
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded") { CharSet = "utf-8" };
var httpClient = new HttpClient();
HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result;
if (response.IsSuccessStatusCode)
{
string result = response.Content.ReadAsStringAsync().Result;
return result;
}
return null;
}
///
/// 删除API
///
///
///
public static bool KongDeleteResponse(string url)
{
if (url.StartsWith("https"))
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
var httpClient = new HttpClient();
HttpResponseMessage response = httpClient.DeleteAsync(url).Result;
return response.IsSuccessStatusCode;
}
///
/// 修改或者更改API
///
///
///
///
public static string KongPutResponse(string url, string postData)
{
if (url.StartsWith("https"))
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
HttpContent httpContent = new StringContent(postData);
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded") { CharSet = "utf-8" };
var httpClient = new HttpClient();
HttpResponseMessage response = httpClient.PutAsync(url, httpContent).Result;
if (response.IsSuccessStatusCode)
{
string result = response.Content.ReadAsStringAsync().Result;
return result;
}
return null;
}
///
/// 检索API
///
///
///
public static string KongSerchResponse(string url)
{
if (url.StartsWith("https"))
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
var httpClient = new HttpClient();
HttpResponseMessage response = httpClient.GetAsync(url).Result;
if (response.IsSuccessStatusCode)
{
string result = response.Content.ReadAsStringAsync().Result;
return result;
}
return null;
}
}到此这篇关于C#使用HttpClient获取HTTP请求与响应的文章就介绍到这了。希望对大家的学习有所帮助，也希望大家多多支持脚本之家。