C# 采集网页，可能需先登陆

Question

采集网址：https://www.taoguba.com.cn/moreTopic?userID=252069

需给出demo，测试通过后马上采纳
哪位能帮忙看看呀

Answer 1

思路就是抓包，然后分析有什么cookie带上，你也可以自己试试看。
你的用户名密码是什么，我给你写了邮件，你回一下。

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;

namespace Q750623
{
    class Program
    {
        public static string PostRequest(string postData, string requestUrlString, CookieContainer cookie)
        {
            ASCIIEncoding encoding = new ASCIIEncoding();
            byte[] data = encoding.GetBytes(postData);
            HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create(requestUrlString);
            myRequest.Method = "POST";
            myRequest.ContentType = "application/x-www-form-urlencoded";
            myRequest.ContentLength = data.Length;
            myRequest.CookieContainer = cookie;
            Stream newStream = myRequest.GetRequestStream();
            newStream.Write(data, 0, data.Length);
            newStream.Close();
            HttpWebResponse myResponse = (HttpWebResponse)myRequest.GetResponse();
            StreamReader reader = new StreamReader(myResponse.GetResponseStream(), Encoding.UTF8);
            return reader.ReadToEnd();
        }

        public static string GetRequest(string requestUrlString, CookieContainer cookie)
        {
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(requestUrlString);

            request.Method = "GET";
            request.CookieContainer = cookie;
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
            return reader.ReadToEnd();
        }

        public static string PostLogin(string postData, string requestUrlString, ref CookieContainer cookie)
        {
            byte[] data = Encoding.UTF8.GetBytes(postData);
            //向服务端请求
            HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create(requestUrlString);
            myRequest.Method = "POST";
            myRequest.ContentType = "application/x-www-form-urlencoded";
            myRequest.ContentLength = data.Length;
            myRequest.CookieContainer = new CookieContainer();
            Stream newStream = myRequest.GetRequestStream();
            newStream.Write(data, 0, data.Length);
            newStream.Close();
            //将请求的结果发送给客户端(界面、应用)
            HttpWebResponse myResponse = (HttpWebResponse)myRequest.GetResponse();
            cookie.Add(myResponse.Cookies);
            StreamReader reader = new StreamReader(myResponse.GetResponseStream(), Encoding.UTF8);
            return reader.ReadToEnd();
        }

        static void Main(string[] args)
        {
            CookieContainer cookie = new CookieContainer();
            PostRequest("userName=用户名&password=密码&checkCode=&save=Y&url=https%3A%2F%2Fwww.taoguba.com.cn%2FmoreTopic%3FuserID%3D252069", "https://sso.taoguba.com.cn/web/login/submit", cookie);
            string s = GetRequest("https://www.taoguba.com.cn/moreTopic?userID=252069", cookie);
            Console.WriteLine(s);
        }
    }
}