Server/HostService/SpiderHelper.cs

这里只显示每个文件的开头 60 行。登录后可以解锁完整代码。

微信扫码关注公众号，代码整理好后第一时间通知你。

using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using System.Collections.Generic;
using System.Security.Cryptography;
using System.Text.Json;
using static AntDesign.IconType;

namespace OnlineOpinionAnalysis.Server.HostService
{
	public class SpiderHelper
	{
		public const string UserAgent =
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36";
		public static HttpClientHandler Handler { get; }
		public static HttpClient HttpClient { get; }
		public static HtmlParser HtmlParser { get; }



		static SpiderHelper()
		{
			Handler = new HttpClientHandler();
			HttpClient = new HttpClient(Handler);
			HttpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.57");
			//HttpClient.DefaultRequestHeaders.Add("X-Requested-With","XMLHttpRequest");
			//HttpClient.DefaultRequestHeaders.Add("Host","index.baidu.com");
			//HttpClient.DefaultRequestHeaders.Add("Connection","keep-alive");
			//HttpClient.DefaultRequestHeaders.Add("Cookie", "BDUSS=kxuSTcyb1d1Z3NQb2JlemQ5dU5BaTZCNkJQUHR2a09ZSURDam95ZVZJbGhZa3BrRVFBQUFBJCQAAAAAAAAAAAEAAADIErhPTUNfWkhBTkdaSEFORwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGHVImRh1SJkf");


			HtmlParser = new();
		}

		public static async Task<IHtmlDocument> GetHtmlDocumentAsync(string url)
		{
			try
			{
				var responseHtml = await HttpClient.GetStringAsync(url);
				var document = await HtmlParser.ParseDocumentAsync(responseHtml);
				
				return document;
			}
			catch (HttpRequestException e)
			{
				Console.WriteLine($"Http请求异常:{e.Message}");
				return null;
			}
			catch (NullReferenceException e)
			{
				Console.WriteLine($"返回值为空:{e.Message}");
				return null;
			}
		}


		public static async Task<string> GetBaiduInfo()
		{
       
            string keyword = "mc";