通过HttpHelper 批量下载数据，出问题了

zi198811290 · 发表于 2013-10-25 23:14:56

网址：联-通-网址 http://uac.10010.com/portal/mallLogin.jsp实现目的：通过登录网站批量获取所有账户的通话记录。
实现步骤：
1. 通过浏览器跟踪得到验证地址

http://uac.10010.com/portal/Service/MallLogin?callback=jsonp1382685209388&userName=账户&password=密码&pwdType=01&productType=02&verifyCode=验证码&redirectType=MALLLOGIN&areaCode=390&redirectURL=&rememberMe=0&arrcity=%E5%8E%A6%E9%97%A8&iVersion=
2.进入首页
http://www.10010.com
(这里跨域跳转，可能跳转到这一步就Cookie已经出错了)
3.进入页面（所有商品分类——查话费——通话详单）
http://iservice.10010.com/ehallService/static/login/r?menuid=000100030001
（服务器进行跳转 http://iservice.10010.com/index_.html）
(跨域跳转，进行代码测试时发现得到的Cookie值是空，但是通过Chrome浏览器跟踪是有Cookie，不明白是哪里的问题造成的)
4.查询通话记录，并下载Execl

问题：
因为很少做过这个这种开发，不清楚问题是哪里出了问题，实在不行就用WebBrowser控件。
如果采用JS记录Cookie那估计就有问题，但是通过浏览器跟踪并没有这个情况，经验少。

代码片段

using System;
using System.Collections.Generic;
using System.Text;
using System.Drawing;
using System.IO;
namespace Phone.Common
{
/// <summary>
///
/// </summary>
public class Unicom
{
HttpHelper http = new HttpHelper();//定义Http对象
private string cookie = "";
public string GetCookie
{
get { return cookie; }
}
public HttpItem GetItem()
{
HttpItem item = new HttpItem() {
URL = "http://www.sufeinet.com",//URL 必需项
Encoding = null,//编码格式（utf-8,gb2312,gbk）可选项默认类会自动识别
//Encoding = Encoding.Default,
Method = "get",//URL 可选项默认为Get
Timeout = 100000,//连接超时时间可选项默认为100000
ReadWriteTimeout = 30000,//写入Post数据超时时间可选项默认为30000
//IsToLower = false,//得到的HTML代码是否转成小写可选项默认转小写
Cookie = cookie,//字符串Cookie 可选项
UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",//用户的浏览器类型，版本，操作系统可选项有默认值
Accept = "text/html, application/xhtml+xml, */*",// 可选项有默认值
ContentType = "text/html",//返回类型可选项有默认值
Referer = "http://www.10010.com/",//来源URL 可选项
Allowautoredirect = true,//是否根据３０１跳转可选项
//CerPath = "d:\\123.cer",//证书绝对路径可选项不需要证书时可以不写这个参数
Connectionlimit = 1024,//最大连接数可选项默认为1024
//Postdata = "C:\\PERKYSU_20121129150608_ScrubLog.txt",//Post数据可选项GET时不需要写
PostDataType = PostDataType.FilePath,//默认为传入String类型，也可以设置PostDataType.Byte传入Byte类型数据
//ProxyIp = "192.168.1.105：8015",//代理服务器ID 端口可以直接加到后面以：分开就行了可选项不需要代理时可以不设置这三个参数
//ProxyPwd = "123456",//代理服务器密码可选项
//ProxyUserName = "administrator",//代理服务器账户名可选项
ResultType = ResultType.Byte,//返回数据类型，是Byte还是String
//PostdataByte = System.Text.Encoding.Default.GetBytes("测试一下"),//如果PostDataType为Byte时要设置本属性的值
CookieCollection = new System.Net.CookieCollection(),//可以直接传一个Cookie集合进来
};
return item;
}
/// <summary>
/// 获取验证码图片
/// </summary>
/// <returns></returns>
public Image GetVerifyImage()
{
HttpItem item = GetItem();
item.URL = "http://uac.10010.com/portal/Service/CreateImage";
HttpResult result = http.GetHtml(item);
cookie = result.Cookie;
using (MemoryStream ms = new MemoryStream(result.ResultByte))
{
Image outputImg = Image.FromStream(ms);
return outputImg;
}
}
/// <summary>
/// 登录网站信息
/// </summary>
/// <param name="name"></param>
/// <param name="pwd"></param>
/// <param name="verifyCode"></param>
/// <returns></returns>
public string Logo(string name, string pwd, string verifyCode)
{
string url = string.Format("http://uac.10010.com/portal/Service/MallLogin?callback=jsonp1382685209388&userName={0}&password={1}&pwdType=01&productType=02&verifyCode={2}&redirectType=MALLLOGIN&areaCode=390&redirectURL=&rememberMe=0&arrcity=%E5%8E%A6%E9%97%A8&iVersion=", name, pwd, verifyCode);
HttpItem item = GetItem();
item.URL = url;
item.Cookie = cookie;
HttpResult result = http.GetHtml(item);
cookie = result.Cookie;
return result.Html;
}
/// <summary>
/// 获取电话清单源代码
/// </summary>
/// <returns></returns>
public string TelephoneList()
{
string url = "http://iservice.10010.com/ehallService/static/login/r?menuid=000100030001";
HttpItem item = GetItem();
item.URL = url;
item.Cookie = cookie;
HttpResult result = http.GetHtml(item);
//程序运行并没有发生异常，这里的Cookie是空值
cookie = result.Cookie;
return result.Html;
}
public string SetUserSessionQ()
{
string url = "http://iservice.10010.com/ehallService/static/queryMonth/checkmapExtraParam/0001";
HttpItem item = GetItem();
item.Method = "post";
item.URL = url;
HttpResult result = http.GetHtml(item);
cookie = result.Cookie;
return result.Html;
}
/// <summary>
/// 设置日期Session
/// </summary>
/// <returns></returns>
public string SetUserSession()
{
string url = "http://iservice.10010.com/ehallService/static/queryMonth/execute2/YHgetMonths/QUERY_paramSession.processData/QUERY_paramSession_Data/000100030001/undefined/2013-10-23/2013-10-23/0?_=1382695343078&menuid=000100030001";
HttpItem item = GetItem();
item.Method = "post";
item.URL = url;
HttpResult result = http.GetHtml(item);
cookie = result.Cookie;
return result.Html;
}
/// <summary>
///
/// </summary>
public void DomnExecl()
{
string url = "http://iservice.10010.com/ehallService/ToExcel.jsp?type=02&_=1382695345299";
HttpItem item = GetItem();
item.Cookie = "iservice=g6wpSqSc81QFTrn1n0pK4rN5Nx4ZTN0MZGVZ6PZQh1njcsnb6R1n!880843110!915873891;BIGipServerPOOL_W2PORTAL_APP=252646410.17183.0000; BIGipServerPOOL_W2PORTAL_WEB=286198282.20480.0000; mallcity=11|110; usst=13827017911071655; uvst=13827863911079608; HJ5107VISITSNID=0; piw=%7B%22login_name%22%3A%220592-31***76%22%2C%22nickName%22%3A%220592-31***76%22%2C%22rme%22%3Anull%2C%22verifyState%22%3A%22%22%7D; JUT=gH7fBwlrjvOE1rkAy8jVbS1qS3PSdCWl/4RTefnLZoqNaSu0pU4YY9+ZQRs8RbTrXBR1xCjVmeYr2lIRq79PUKpOZrk8UVwX4da0QgB3ThxOB8f35Tu+5nSmhP4wd7fIqz5fGfWyds4q5h3mv02L9yzTggIoUT2z6dh/CWM8PG8NiwGkTvkHNyjbJCYepFzcsL+dnP8SaobRLAcjmziinVF71JS6WM+kvCF+zexumrEEJEKHcC4hYbXQe+gwcRv8q99e9NPlmxkOS0YH/KAofma+OBGE0H+FbpqdAKrzcKiNEqCUEV1b1UoAg8GbSNMQ3TSPdEtqyh79pqJ1NvhdXREzILzJWlxRD49jLaWX21voOd1Zfg7a17JonnYRdXObYugeHEvdoz4n7WGK1MLYwKy3Ou98CyKylL/H9O7bbpvK0dtFtme0qKiIX40aDAkrypUByNZffAqylvvF+ud12lEDjdOErtJTRlBnJGNpLdgwFG2zBPrzasb7AL4EqSSwQ8MqwTZ40m46pmZUNzam7Q==; WT_FPC=id=22db68b473d40af46041382699991066:lv=1382700795166:ss=1382699991066; __utma=231252639.498526238.1382699991.1382699991.1382699991.1; __utmb=231252639.16.10.1382699991; __utmc=231252639; __utmz=231252639.1382699991.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmv=231252639.Beijing; _n3fa_cid=844adb49bc934b66f26416d938fe6bfa; _n3fa_lvt_a9e72dfe4a54a20c3d6e671b3bad01d9=1382700795; _n3fa_lpvt_a9e72dfe4a54a20c3d6e671b3bad01d9=1382700795; Hm_lvt_9208c8c641bfb0560ce7884c36938d9d=1382699992; Hm_lpvt_9208c8c641bfb0560ce7884c36938d9d=1382700795; HJ5107ONCEVISITID8a949ccd3f466469013f4664fe540001=0113951932336130.01832340145483613; MI=000100030001; MJ=WT/WT_MENU_001/038/390/032; MP=WT/WT_MENU_004/038/390/032; PROVINCECODE=038; USERNUM=3116576";
item.URL = url;
HttpResult result = http.GetHtml(item);
FileStream fs = new FileStream(@"D:\123.xls", FileMode.Create, FileAccess.Write);
fs.Write(result.ResultByte, 0, result.ResultByte.Length);
fs.Flush();
fs.Close();
}
public string GetUrlHtml(string url)
{
HttpItem item = GetItem();
item.URL = string.Format("http://uac.10010.com/portal/Service/MallLogin?callback=jsonp1382685209388&userName=3116576&password=789987&pwdType=01&productType=02&verifyCode={0}&redirectType=MALLLOGIN&areaCode=390&redirectURL=&rememberMe=0&arrcity=%E5%8E%A6%E9%97%A8&iVersion=", url);
HttpResult result = http.GetHtml(item);
return result.Html;
}
public Unicom()
{
}
/// <summary>
/// 进入首页
/// </summary>
/// <returns></returns>
public string Index()
{
string url = "http://www.10010.com/";
HttpItem item = GetItem();
item.URL = url;
item.Cookie = cookie;
HttpResult result = http.GetHtml(item);
cookie = result.Cookie;
return result.Html;
}
}
}

复制代码

我用VS2010写了测试代码
http://pan.baidu.com/s/1wyUqw

cload · 发表于 2013-10-26 18:17:55

Cookie值是变化着的，看你的代码中怎么固定一部分了？

zi198811290 · 发表于 2013-10-28 11:25:00

cload 发表于 2013-10-26 18:17
Cookie值是变化着的，看你的代码中怎么固定一部分了？

本来是没有的
我当时为了验证里面的数据，才固定一些数据的，主要是网站里面用到一些Session，我没有找到对应的Session ID，所有出了问题

		自动登录	找回密码
密码			马上注册

[其他] 通过HttpHelper 批量下载数据，出问题了