forked from youbl/Beinet.cn.DrivingTest
-
Notifications
You must be signed in to change notification settings - Fork 0
/
GetQuestion.cs
375 lines (343 loc) · 14.9 KB
/
GetQuestion.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Net.Security;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Json;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Windows.Forms;
using Beinet.cn.DrivingTest;
namespace DrivingTest
{
/// <summary>
/// 抓取考试题目的代码窗体
/// </summary>
public partial class GetQuestion : Form
{
#region 属性与变量
/// <summary>
/// 考试题目所在目录和图片
/// </summary>
internal static readonly string QuestionDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"jk\");//@"e:\jk\{0}";
/// <summary>
/// 默认的保存编码格式
/// </summary>
internal static Encoding GB2312 = Encoding.GetEncoding("GB2312");
/// <summary>
/// 考题里的分隔符,必须与抓取程序一致
/// </summary>
internal const string SPLIT_STR = "======================";
/// <summary>
/// C1驾照的题目总数,如果要包含大型车,请改成2540题,再去抓取题目
/// </summary>
internal const int QUESTION_NUM = 973;// 2011年C1考题总数是725;
/// <summary>
/// 访问网络用的Cookie容器
/// </summary>
internal static CookieContainer Cookies = new CookieContainer();
// 2013年以前的url地址
//private const string _starturl = "http://wz.jxedt.com/test/201004abcd/201004-sxtest.asp?type=c";
//private const string _quesurl =
// "http://wz.jxedt.com/test/201004abcd/Ask.asp?myTotal=725<ype=c&myID={0}&myOrder={0}";
private const string _starturl = "http://wz.jxedt.com/test/2014abcd/sxlx.asp?type=c";
private const string _quesurl =
"http://wz.jxedt.com/test/2014abcd/ajax.asp?r=0.555714223572155&index={0}";
#endregion
public GetQuestion()
{
InitializeComponent();
if (!Directory.Exists(QuestionDir))
{
Directory.CreateDirectory(QuestionDir);
}
bool isok;
// 得到Cookie
GetPage(_starturl, out isok, Cookies, encoding: GB2312);
for (int i = 1; i <= Form1.QUESTION_NUM; i++)
{
var url = string.Format(_quesurl, i);
var html = GetPage(url, out isok, Cookies, encoding: GB2312);
if (!isok)
{
textBox1.Text = html;
return;
}
//var jsonfile = fileName + i.ToString() + ".txt";
//using (var sw = new StreamWriter(jsonfile, false, enc))
//{
// sw.Write(html);
//}
//continue;
Question question;
try
{
question = JsonToObject<Question>(html);
}
catch (Exception exp)
{
textBox1.Text = html + "\r\n\r\n" + exp;
return;
}
var quesimg = string.Empty; // 问题图片列表
if (!string.IsNullOrEmpty(question.imageurl))//question.imageurl
{
// 大图
//http://ww3.sinaimg.cn/mw600/5eb4d75agw1e2918wh62bj.jpg
// 小图
//http://ww3.sinaimg.cn/mw240/5eb4d75agw1e2918wh62bj.jpg
string bigUrl = "http://ww3.sinaimg.cn/mw600/" + question.sinaimg;
// 问题包括图片时,下载
var tmp = DownImgs(bigUrl, i);//(question.imageurl);
quesimg += tmp;
}
int rightAnswer = question.ta; // 答案序号
//if (rightAnswer.Length > 1)
//{
// // 用于断点,判断有没有多选题用
//}
List<string> answerOpn = new List<string>();
// 答案选项
answerOpn.Add(question.a);
answerOpn.Add(question.b);
if (!string.IsNullOrEmpty(question.c))
answerOpn.Add(question.c);
if (!string.IsNullOrEmpty(question.d))
answerOpn.Add(question.d);
var rightAnswerDesc = question.bestanswer; // 答案简要说明
var file = QuestionDir + i.ToString() + ".txt";
using (var sw = new StreamWriter(file, false, GB2312))
{
sw.WriteLine(question.question); // 问题
sw.WriteLine(Form1.SPLIT_STR);
sw.WriteLine(quesimg); // 问题图片列表,以逗号分隔
sw.WriteLine(Form1.SPLIT_STR);
sw.WriteLine(rightAnswer); // 答案序号
sw.WriteLine(Form1.SPLIT_STR);
sw.WriteLine(rightAnswerDesc); // 答案简要说明
foreach (var opn in answerOpn)
{
sw.WriteLine(Form1.SPLIT_STR);
sw.WriteLine(opn); // 答案选项
}
}
}
textBox1.Text = "全部完成";
}
string DownImgs(string imeurl, int id)
{
string ret = string.Empty;
var imgname = id.ToString() + imeurl.Substring(imeurl.LastIndexOf('.'));
var imgfile = QuestionDir + imgname;
if (!File.Exists(imgfile))
DownloadFile(imeurl, imgfile, Cookies);
ret += imgname + ",";
return ret;
}
/// <summary>
/// 抓取页面
/// </summary>
/// <param name="url">要抓取的网址</param>
/// <param name="isok">抓取结果,成功还是失败</param>
/// <param name="cookieContainer">要使用的cookie</param>
/// <param name="param">参数</param>
/// <param name="HttpMethod">GET POST</param>
/// <param name="encoding">编码格式,默认UTF-8</param>
/// <param name="showHeader">返回内容是否包括头信息</param>
/// <param name="userName">网页登录名</param>
/// <param name="password">登录密码</param>
/// <returns></returns>
static string GetPage(string url, out bool isok, CookieContainer cookieContainer,
string param = null, string HttpMethod = null,
Encoding encoding = null,
bool showHeader = false, string userName = null, string password = null)
{
isok = true;
if (encoding == null)
encoding = Encoding.UTF8;
bool isGet = (string.IsNullOrEmpty(HttpMethod) ||
HttpMethod.Equals("GET", StringComparison.OrdinalIgnoreCase));
#region Get方式,且有参数时,把参数拼接到Url后面
if (isGet && !string.IsNullOrEmpty(param))
{
// 删除网址后面的#号
if (url.IndexOf('#') >= 0)
url = url.Substring(0, url.IndexOf('#'));
if (url.IndexOf('?') < 0)
{
url += "?" + param;
}
else
{
url += "&" + param;
}
}
#endregion
// 访问Https网站时,加上特殊处理,用于处理证书有问题的网站
if (url.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
if (!string.IsNullOrEmpty(userName) || !string.IsNullOrEmpty(password))
request.Credentials = new NetworkCredential(userName, password);
#region 加Cookie
request.CookieContainer = cookieContainer;
// request.CookieContainer.SetCookies(new Uri(url), "aaa=bbb&ccc=ddd");// 必须一次全部加入Cookie
#endregion
//request.Referer = url;
request.AllowAutoRedirect = false; //禁止自动转向,静态化目标页面发生转向是因为出了异常
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1;)";
//request.KeepAlive = true;
//request.Timeout = 100000; // 设置超时时间,默认值为 100,000 毫秒(100 秒)。
//request.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, * /*";
request.Headers.Add("Accept-Encoding", "gzip, deflate");
request.Accept = "* /*";
request.Headers.Add("Accept-Language", "zh-cn");
if (isGet)
{
request.Method = "GET";
//request.ContentType = "text/html";
}
else
{
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
// 设置提交的数据
if (!string.IsNullOrEmpty(param))
{
request.ContentLength = param.Length;
Stream newStream = request.GetRequestStream();
// 把数据转换为字节数组
byte[] l_data = encoding.GetBytes(param);
newStream.Write(l_data, 0, l_data.Length);
newStream.Close();
}
}
HttpWebResponse response;
try
{
response = request.GetResponse() as HttpWebResponse;
}
catch (Exception exp)
{
isok = false;
return "返回错误:" + exp;
}
if (response == null)
{
return "Response is null";
}
if (response.StatusCode == HttpStatusCode.OK)
{
Stream stream;
using (stream = response.GetResponseStream())
{
if (response.ContentEncoding.ToLower().Contains("gzip"))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))
{
stream = new DeflateStream(stream, CompressionMode.Decompress);
}
if (stream == null)
{
return "stream is null";
}
using (StreamReader sr = new StreamReader(stream, encoding))
{
string html = sr.ReadToEnd();
if (showHeader)
html = "请求头信息:\r\n" + request.Headers + "\r\n\r\n响应头信息:\r\n" + response.Headers + "\r\n" + html;
return html;
}
}
}
return "远程服务器返回代码不为200," +
response.StatusCode + "," + response.StatusDescription;
}
/// <summary>
/// 用于访问Https站点时,证书有问题,始终返回true
/// </summary>
/// <param name="sender"></param>
/// <param name="certificate"></param>
/// <param name="chain"></param>
/// <param name="errors"></param>
/// <returns></returns>
static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{
// Always accept
//Console.WriteLine("accept" + certificate.GetName());
return true; //总是接受
}
/// <summary>
/// 下载文件
/// </summary>
/// <param name="downloadUrl"></param>
/// <param name="savePath"></param>
/// <param name="cookieContainer"></param>
void DownloadFile(string downloadUrl, string savePath, CookieContainer cookieContainer)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(downloadUrl);
request.AllowAutoRedirect = true;
request.CookieContainer = cookieContainer;
byte[] buffer = new byte[1000000];
using (var response = request.GetResponse())
using (var stream = new FileStream(savePath,
FileMode.CreateNew, FileAccess.Write, FileShare.None))
using (var responseStream = response.GetResponseStream())
{
if (responseStream == null)
{
return;
}
long s = stream.Length, l = response.ContentLength;
int read;
while (s < l && (read = responseStream.Read(buffer, 0, buffer.Length)) != 0)
{
stream.Write(buffer, 0, read);
stream.Flush();
s += read;
//var percent = (s*100/(decimal) l).ToString("N");
//OperationLabelMethod(labProcess, "下载进度:" + percent + "% " + "点这里取消");
}
responseStream.Close();
//OperationLabelMethod(labProcess, null); // 隐藏进度
}
}
///// <summary>
///// 对象生成Json串
///// </summary>
///// <typeparam name="T"></typeparam>
///// <param name="source"></param>
///// <returns></returns>
//static string StringToJson<T>(T source)
//{
// DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(T));
// using (MemoryStream ms = new MemoryStream())
// {
// jsonSerializer.WriteObject(ms, source);
// StringBuilder sb = new StringBuilder();
// sb.Append(Encoding.UTF8.GetString(ms.ToArray()));
// return sb.ToString();
// }
//}
/// <summary>
/// Json串生成对象
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="source"></param>
/// <returns></returns>
static T JsonToObject<T>(string source)
{
DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(T));
using (MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(source)))
{
T obj = (T)jsonSerializer.ReadObject(ms);
return obj;
}
}
}
}