1 using System; 2 using System.Collections.Generic; 3 using System.Diagnostics; 4 using System.IO; 5 using System.IO.Compression; 6 using System.Linq; 7 using System.Net; 8 using System.Net.Security; 9 using System.Runtime.InteropServices; 10 using System.Security.Cryptography.X509Certificates; 11 using System.Text; 12 using System.Text.RegularExpressions; 13 using System.Threading.Tasks; 14 15 namespace ClientApp 16 {/// <summary> 17 /// Http连接操作帮助类 18 /// </summary> 19 public class HttpHelper 20 { 21 private const int ConnectionLimit = 100; 22 //编码 23 private Encoding _encoding = Encoding.Default; 24 //浏览器类型 25 private string[] _useragents = new string[]{ 26 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36', 27 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)', 28 'Mozilla/5.0 (Windows NT 6.1; rv:36.0) Gecko/20100101 Firefox/36.0', 29 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0' 30 }; 31 32 private String _useragent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36'; 33 //接受类型 34 private String _accept = 'text/html, application/xhtml+xml, application/xml, */*'; 35 //超时时间 36 private int _timeout = 30 * 1000; 37 //类型 38 private string _contenttype = 'application/x-www-form-urlencoded'; 39 //cookies 40 private String _cookies = ''; 41 //cookies 42 private CookieCollection _cookiecollection; 43 //custom heads 44 private Dictionary<string, string> _headers = new Dictionary<string, string>(); 45 46 public HttpHelper() 47 { 48 _headers.Clear(); 49 //随机一个useragent 50 _useragent = _useragents[new Random().Next(0, _useragents.Length)]; 51 //解决性能问题? 52 ServicePointManager.DefaultConnectionLimit = ConnectionLimit; 53 } 54 55 public void InitCookie() 56 { 57 _cookies = ''; 58 _cookiecollection = null; 59 _headers.Clear(); 60 } 61 62 /// <summary> 63 /// 设置当前编码 64 /// </summary> 65 /// <param name='en'></param> 66 public void SetEncoding(Encoding en) 67 { 68 _encoding = en; 69 } 70 71 /// <summary> 72 /// 设置UserAgent 73 /// </summary> 74 /// <param name='ua'></param> 75 public void SetUserAgent(String ua) 76 { 77 _useragent = ua; 78 } 79 80 public void RandUserAgent() 81 { 82 _useragent = _useragents[new Random().Next(0, _useragents.Length)]; 83 } 84 85 public void SetCookiesString(string c) 86 { 87 _cookies = c; 88 } 89 90 /// <summary> 91 /// 设置超时时间 92 /// </summary> 93 /// <param name='sec'></param> 94 public void SetTimeOut(int msec) 95 { 96 _timeout = msec; 97 } 98 99 public void SetContentType(String type)100 {101 _contenttype = type;102 }103 104 public void SetAccept(String accept)105 {106 _accept = accept;107 }108 109 /// <summary>110 /// 添加自定义头111 /// </summary>112 /// <param name='key'></param>113 /// <param name='ctx'></param>114 public void AddHeader(String key, String ctx)115 {116 //_headers.Add(key,ctx);117 _headers[key] = ctx;118 }119 120 /// <summary>121 /// 清空自定义头122 /// </summary>123 public void ClearHeader()124 {125 _headers.Clear();126 }127 128 /// <summary>129 /// 获取HTTP返回的内容130 /// </summary>131 /// <param name='response'></param>132 /// <returns></returns>133 private String GetStringFromResponse(HttpWebResponse response)134 {135 String html = '';136 try137 {138 Stream stream = response.GetResponseStream();139 StreamReader sr = new StreamReader(stream, Encoding.UTF8);140 html = sr.ReadToEnd();141 142 sr.Close();143 stream.Close();144 }145 catch (Exception e)146 {147 Trace.WriteLine('GetStringFromResponse Error: ' + e.Message);148 }149 150 return html;151 }152 153 /// <summary>154 /// 检测证书155 /// </summary>156 /// <param name='sender'></param>157 /// <param name='certificate'></param>158 /// <param name='chain'></param>159 /// <param name='errors'></param>160 /// <returns></returns>161 private bool CheckCertificate(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)162 {163 return true;164 }165 166 /// <summary>167 /// 发送GET请求168 /// </summary>169 /// <param name='url'></param>170 /// <returns></returns>171 public String HttpGet(String url)172 {173 return HttpGet(url, url);174 }175 176 177 /// <summary>178 /// 发送GET请求179 /// </summary>180 /// <param name='url'></param>181 /// <param name='refer'></param>182 /// <returns></returns>183 public String HttpGet(String url, String refer)184 {185 String html;186 try187 {188 ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckCertificate);189 HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);190 request.UserAgent = _useragent;191 request.Timeout = _timeout;192 request.ContentType = _contenttype;193 request.Accept = _accept;194 request.Method = 'GET';195 request.Referer = refer;196 request.KeepAlive = true;197 request.AllowAutoRedirect = true;198 request.UnsafeAuthenticatedConnectionSharing = true;199 request.CookieContainer = new CookieContainer();200 //据说能提高性能201 //request.Proxy = null;202 if (_cookiecollection != null)203 {204 foreach (Cookie c in _cookiecollection)205 {206 c.Domain = request.Host;207 }208 209 request.CookieContainer.Add(_cookiecollection);210 }211 212 foreach (KeyValuePair<String, String> hd in _headers)213 {214 request.Headers[hd.Key] = hd.Value;215 }216 217 HttpWebResponse response = (HttpWebResponse)request.GetResponse();218 html = GetStringFromResponse(response);219 if (request.CookieContainer != null)220 {221 response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);222 }223 224 if (response.Cookies != null)225 {226 _cookiecollection = response.Cookies;227 }228 if (response.Headers['Set-Cookie'] != null)229 {230 string tmpcookie = response.Headers['Set-Cookie'];231 _cookiecollection.Add(ConvertCookieString(tmpcookie));232 }233 234 response.Close();235 return html;236 }237 catch (Exception e)238 {239 Trace.WriteLine('HttpGet Error: ' + e.Message);240 return String.Empty;241 }242 }243 244 /// <summary>245 /// 获取MINE文件246 /// </summary>247 /// <param name='url'></param>248 /// <returns></returns>249 public Byte[] HttpGetMine(String url)250 {251 Byte[] mine = null;252 try253 {254 ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckCertificate);255 HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);256 request.UserAgent = _useragent;257 request.Timeout = _timeout;258 request.ContentType = _contenttype;259 request.Accept = _accept;260 request.Method = 'GET';261 request.Referer = url;262 request.KeepAlive = true;263 request.AllowAutoRedirect = true;264 request.UnsafeAuthenticatedConnectionSharing = true;265 request.CookieContainer = new CookieContainer();266 //据说能提高性能267 request.Proxy = null;268 if (_cookiecollection != null)269 {270 foreach (Cookie c in _cookiecollection)271 c.Domain = request.Host;272 request.CookieContainer.Add(_cookiecollection);273 }274 275 foreach (KeyValuePair<String, String> hd in _headers)276 {277 request.Headers[hd.Key] = hd.Value;278 }279 280 HttpWebResponse response = (HttpWebResponse)request.GetResponse();281 Stream stream = response.GetResponseStream();282 MemoryStream ms = new MemoryStream();283 284 byte[] b = new byte[1024];285 while (true)286 {287 int s = stream.Read(b, 0, b.Length);288 ms.Write(b, 0, s);289 if (s == 0 || s < b.Length)290 {291 break;292 }293 }294 mine = ms.ToArray();295 ms.Close();296 297 if (request.CookieContainer != null)298 {299 response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);300 }301 302 if (response.Cookies != null)303 {304 _cookiecollection = response.Cookies;305 }306 if (response.Headers['Set-Cookie'] != null)307 {308 _cookies = response.Headers['Set-Cookie'];309 }310 311 stream.Close();312 stream.Dispose();313 response.Close();314 return mine;315 }316 catch (Exception e)317 {318 Trace.WriteLine('HttpGetMine Error: ' + e.Message);319 return null;320 }321 }322 323 /// <summary>324 /// 发送POST请求325 /// </summary>326 /// <param name='url'></param>327 /// <param name='data'></param>328 /// <returns></returns>329 public String HttpPost(String url, String data)330 {331 return HttpPost(url, data, url,null);332 }333 334 /// <summary>335 /// 发送POST请求336 /// </summary>337 /// <param name='url'></param>338 /// <param name='data'></param>339 /// <param name='refer'></param>340 /// <returns></returns>341 public String HttpPost(String url, String data, String refer,string cookie)342 {343 String html;344 try345 {346 ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckCertificate);347 HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);348 request.UserAgent = _useragent;349 request.Timeout = _timeout;350 request.Referer = refer;351 request.ContentType = _contenttype;352 request.Accept = _accept;353 request.Method = 'POST';354 request.KeepAlive = true;355 request.AllowAutoRedirect = true;356 357 request.CookieContainer = new CookieContainer();358 if (!string.IsNullOrEmpty(cookie))359 {360 _cookiecollection = this.ConvertCookieString(cookie);361 }362 //据说能提高性能363 request.Proxy = null;364 365 if (_cookiecollection != null)366 {367 foreach (Cookie c in _cookiecollection)368 {369 c.Domain = request.Host;370 if (c.Domain.IndexOf(':') > 0)371 c.Domain = c.Domain.Remove(c.Domain.IndexOf(':'));372 }373 request.CookieContainer.Add(_cookiecollection);374 }375 376 foreach (KeyValuePair<String, String> hd in _headers)377 {378 request.Headers[hd.Key] = hd.Value;379 }380 byte[] buffer = _encoding.GetBytes(data.Trim());381 request.ContentLength = buffer.Length;382 request.GetRequestStream().Write(buffer, 0, buffer.Length);383 request.GetRequestStream().Close();384 385 HttpWebResponse response = (HttpWebResponse)request.GetResponse();386 html = GetStringFromResponse(response);387 if (request.CookieContainer != null)388 {389 response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);390 }391 if (response.Cookies != null)392 {393 _cookiecollection = response.Cookies;394 }395 if (response.Headers['Set-Cookie'] != null)396 {397 string tmpcookie = response.Headers['Set-Cookie'];398 _cookiecollection.Add(ConvertCookieString(tmpcookie));399 }400 401 response.Close();402 return html;403 }404 catch (Exception e)405 {406 Trace.WriteLine('HttpPost Error: ' + e.Message);407 return String.Empty;408 }409 }410 411 412 public string UrlEncode(string str)413 {414 StringBuilder sb = new StringBuilder();415 byte[] byStr = _encoding.GetBytes(str);416 for (int i = 0; i < byStr.Length; i++)417 {418 sb.Append(@'%' + Convert.ToString(byStr[i], 16));419 }420 421 return (sb.ToString());422 }423 424 /// <summary>425 /// 转换cookie字符串到CookieCollection426 /// </summary>427 /// <param name='ck'></param>428 /// <returns></returns>429 private CookieCollection ConvertCookieString(string ck)430 {431 CookieCollection cc = new CookieCollection();432 string[] cookiesarray = ck.Split(';'.ToCharArray(), StringSplitOptions.RemoveEmptyEntries);433 for (int i = 0; i < cookiesarray.Length; i++)434 {435 string[] cookiesarray_2 = cookiesarray[i].Split(','.ToCharArray(), StringSplitOptions.RemoveEmptyEntries);436 for (int j = 0; j < cookiesarray_2.Length; j++)437 {438 string[] cookiesarray_3 = cookiesarray_2[j].Trim().Split('='.ToCharArray());439 if (cookiesarray_3.Length == 2)440 {441 string cname = cookiesarray_3[0].Trim();442 string cvalue = cookiesarray_3[1].Trim();443 if (cname.ToLower() != 'domain' && cname.ToLower() != 'path' && cname.ToLower() != 'expires')444 {445 Cookie c = new Cookie(cname, cvalue);446 cc.Add(c);447 }448 }449 }450 }451 452 return cc;453 }454 455 456 public void DebugCookies()457 {458 Trace.WriteLine('**********************BEGIN COOKIES*************************');459 foreach (Cookie c in _cookiecollection)460 {461 Trace.WriteLine(c.Name + '=' + c.Value);462 Trace.WriteLine('Path=' + c.Path);463 Trace.WriteLine('Domain=' + c.Domain);464 }465 Trace.WriteLine('**********************END COOKIES*************************');466 }467 468 }469 }
