说明主要使用语言&框架:Python&Scrapy源码地址：https://pan.bigdataboy.cn/#/s/6euy爬取思路第一次搜索：获得所有页数url（简化）'https://search.jd.com/Search?keyword={}'methodGETheaders{'user-agent':'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/84.0.4147.105Safari/537.36'}直接请求商品接口，获取商品id&价格url（简化）'https://search.jd.com/s_new.php?keyword={搜索词}&page={页数}&s={分页数}'methodGETheaders{'referer':'https://search.jd.com/Search'}使用商品id，请求商品页面，获得相关信息urlhttps://item.jd.com/{商品id}.htmlmethodGET请求评论接口，获取评论相关数量url）'https://club.jd.com/comment/productCommentSummaries.action?referenceIds={商品id}methodGETheaders{'referer':'https://search.jd.com/Search'}编写爬虫常规设置创建工程&爬虫文件scrapystartprojectjdAppcdjdAppscrapygenspiderjdspiderjd.com修改setting.py文件#伪装#Crawlresponsiblybyidentifyingyourself(andyourwebsite)ontheuser-agentUSER_AGENT='Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/689.36(KHTML,likeGecko)Chrome/99.0.5204.108Safari/637.75'#Obeyrobots.txtrulesROBOTSTXT_OBEY=False#下载延迟DOWNLOAD_DELAY=1#Overridethedefaultrequestheaders:DEFAULT_REQUEST_HEADERS={'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language':'en','referer':'https://search.jd.com/Search',#京东会检测这个}总体架构classJdspiderSpider(scrapy.Spider):name='jdspider'allowed_domains=['jd.com']#爬取的商品good_name=""#第一次请求获取全部商品全部页数search_url='https://search.jd.com/Search?keyword={}'#第一次搜索接口headers={'user-agent':'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/84.0.4147.105Safari/537.36'}#获取商品接口每次请求返回30个goods_url_api='https://search.jd.com/s_new.php?keyword={}&page={}&s={}'#搜索接口#获取页数请求每一页defstart_requests(self):pass#获取每一个商品id，请求商品详情数据defparse(self,response:Response):pass#获取商品详情数据，请求评论信息defget_infor(self,response:Response):pass#得到评论信息，返回itemdefget_comment(self,response:Response):passif__name__=='__main__':#启动fromscrapyimportcmdlinecmdline.execute("scrapyrunspiderjdspider.py".split())start_requests(self)#获取页数请求每一页defstart_requests(self):self.good_name=input("输入爬取的关键词：")r=requests.get(url=self.search_url.format(self.good_name),headers=self.headers)#获取总页数all_pages=re.search(r'<b>1</b><em>/</em><i>(\d*?)</i>',r.text).group(1)logger.info(msg=f"总页数{all_pages}")for_inrange(int(all_pages)*2):#组合每一页的链接goods_url_api=self.goods_url_api.format(self.good_name,_+1,1+25*(_+1-1))logger.info(msg=f"请求链接：{goods_url_api}")#请求每一页的商品yieldRequest(url=goods_url_api,callback=self.parse)parse(self,response:Response)#获取每一个商品id，请求商品详情数据defparse(self,response:Response):item=JdappItem()drug_list:list[Selector]=response.xpath("//div[@id='J_goodsList']/ul/li")for_indrug_list:#商品价格item['good_price']=_.xpath('//div[@class="p-price"]/strong/i/text()').get()#商品iditem['good_id']=_.xpath("@data-sku").get()url=f'https://item.jd.com/{item["good_id"]}.html'logger.info(msg=f"正在爬取商品详情：{url}")yieldRequest(url=url,meta={'item':copy.deepcopy(item),'url':url},callback=self.get_infor,)get_infor(self,response:Response)#获取商品详情数据，请求评论信息defget_infor(self,response:Response):item=response.meta['item']#商品连接item['good_url']=response.urlinfor=dict()#商品介绍parameter:list[str]=response.xpath('//div[@class="p-parameter"]/ul[2]/li/text()').getall()for_inparameter:i=_.split("：")infor[i[0]]=i[1]#规格与包装size_packing_key:list[str]=response.xpath('//div[@class="Ptable"]/div/dl/dl/dt/text()').getall()size_packing_value:list[str]=response.xpath('//div[@class="Ptable"]/div/dl/dl/dd/text()').getall()for_inzip(size_packing_key,size_packing_value):infor[_[0]]=_[1]logger.info(msg=f"商品参数规格信息{infor}")#商品信息item['good_infor']=infor#获取评论url=f'https://club.jd.com/comment/productCommentSummaries.action?referenceIds={item["good_id"]}'logger.info(msg=f"正在爬取商品评论{url}")yieldRequest(url=url,meta={'item':copy.deepcopy(item)},callback=self.get_comment,dont_filter=True)get_comment(self,response:Response)#得到评论信息，返回itemdefget_comment(self,response:Response):item=response.meta['item']comment=json.loads(response.text)#评论数item['comment_number']=comment.get('CommentsCount')[0].get('CommentCountStr')#好评item['good_comment']=comment.get('CommentsCount')[0].get('GoodCountStr')#中评item['medium_comment']=comment.get('CommentsCount')[0].get('GeneralCountStr')#差评item['bad_comment']=comment.get('CommentsCount')[0].get('PoorCountStr')yielditemtip在爬取商品详情页时，需要经常更换IP，不要容易什么都获取不到，源码里的动态转发需要更改为自己的秘钥和订单注意每个Request请求meta的传递方式
爬虫目标博主在假期遇到有家长，拿着老师的布置的作业来打印，结果是如下这样的电子书，还不能下载，所以打算编写爬虫进行下载，发现有书籍的链接全都通过Js加密了，这不能忍，这么能阻止小学生做作业呢。网站：https://mp.zhizhuma.com/book/shelf.htm?id=4872所用模块：#-*-coding:utf-8-*-importrequestsimporttimeimportreimportrandomimportbase64importjsonimporthashlibfromCrypto.CipherimportAESfromosimportmakedirs爬虫结构#获取数据每一页的链接的Jsondefget_encryptedData(ebookId):pass#获取真正的连接，有时效defget_auth_key(data,differenceDate):#下载保存连接defdownload_and_save(datadict,differenceDate):if__name__=='__main__':#书籍的分享链接share_url='https://mp.zhizhuma.com/book/sample2.htm?id=52753&shelfId=4872'ebookId=re.search(r'id=(\d+)',share_url).group(1)#获取书籍的URLencryptedData=get_encryptedData(ebookId)#获取时间戳参数differenceDate=encryptedData.get('timestamp')#创建文件夹makedirs(str(differenceDate))fordatainencryptedData.get('data'):#获取加密的连接pageNo_url=get_auth_key(data=data,differenceDate=differenceDate)print(pageNo_url)#下载书籍download_and_save(pageNo_url,differenceDate=differenceDate)get_encryptedData()通过Js分析，发现这段Js的解密，我们使用Python来实现，逻辑并不复杂。defget_encryptedData(ebookId):url='https://biz.zhizhuma.com/ebookpageservices/queryAllPageByEbookId.do'data={"ebookId":ebookId,"_timestamp":"1586101527","_nonce":"24430072-41ad-48cb-9c7f-880f990c0886","_sign":"975F1339ED050BB789CD51D66E40DD6B",}j=requests.post(url=url,data=data).json().get('encryptedData')#AES解密密钥Js寻找cipher=AES.new("Suj4XDDt3jPsH9Jj".encode(),AES.MODE_ECB)raw_data=cipher.decrypt(base64.decodebytes(bytes(j,encoding='utf8'))).rstrip(b'\x0f').decode("utf-8")[:-1]j_data=json.loads(raw_data)returnj_dataget_auth_key()拼接图片链接后的参数，不带这个参数，获取不到图片auth_key参数的生成，也是模仿Js的来写就好。#获取真正的连接，有时效defget_auth_key(data,differenceDate):#页数pageNo=data.get('pageNo')#解密构造参数imgurl=data.get('imgurl').split('https://cdnyuntisyspro.bookln.cn')[1]uid="0"rand=str(random.random())timestamp=str(int(time.time())-int((int(time.time()*1000)-differenceDate)/1000)+15)sstring=imgurl+"-"+timestamp+"-"+rand+"-"+uid+"-69731cbade6a64b58d60"md5=hashlib.md5()md5.update(sstring.encode())md5hash=md5.hexdigest()authKey='auth_key='+timestamp+"-"+rand+"-"+uid+"-"+md5hashurl="https://cdnyuntisyspro.bookln.cn"+imgurl+"?"+authKeyreturn{"pageNo":pageNo,"url":url}download_and_save()#下载保存defdownload_and_save(datadict,differenceDate):c=requests.get(url=datadict.get('url')).contentwithopen(str(differenceDate)+"/"+str(datadict.get('pageNo'))+".png","wb")asfile:file.write(c)运行可以很容易扩展成多线程。大数据男孩原创，仅用于交流学习之用
爬虫目标所用模块：requests、os、gevent爬虫结构项目结构classWZSpider():def__init__(self):#英雄数据self.herolist="https://pvp.qq.com/web201605/js/herolist.json"#英雄详情页self.infor="https://pvp.qq.com/web201605/herodetail/{}.shtml"#图片链接self.img_url="http://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/{}/{}-bigskin-{}.jpg"#存储英雄详情也链接self.infor_urls=requests.get(url=self.herolist).json()defdownload(self,infor):defrun(self):下载方法把下载方法单独提出，能进行扩展协程，多线程等defdownload(self,infor):ename=infor.get("ename")cname=infor.get("cname")#创建目录path="infor/{}".format(cname)makedirs(path)try:#皮肤名字skins=infor.get("skin_name").split("|")exceptAttributeError:returnelse:forindex,skininenumerate(skins):img_url=self.img_url.format(ename,ename,index+1)img=requests.get(url=img_url)print(cname,skin,img_url)withopen("infor/"+cname+"/"+skin+".jpg","wb")asfile:file.write(img.content)调用defrun(self):glist=[gevent.spawn(self.download,url)forurlinself.infor_urls]gevent.joinall(glist)使用if__name__=='__main__':wzs=WZSpider()wzs.run()
爬虫目标通过抖音的分享链接，获取抖音的原视频（无水印）使用模块：requests、re爬虫结构主要为了最后获取视频时所要的参数classDYSpider():def__init__(self,share_url):#get获取3个参数item_idsmidu_codeself.share_url=share_urlself.item_ids=""self.mid=""self.u_code=""#get获取参数dytk后面大括号需要item_idsself.dytk_url="https://www.iesdouyin.com/share/video/{}/"self.dytk=""#获取信息接口getself.infor_url="https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/"self.headers={"user-agent":"Mozilla/5.0(iPhone;CPUiPhoneOS11_0likeMacOSX)AppleWebKit/604.1.38(KHTML,likeGecko)Version/11.0Mobile/15A372Safari/604.1",}defget_imu(self):passdefget_dytk(self):passdefget_infor(self):passdefrun(self):passget_imu()获取item_idsmidu_code三个参数defget_imu(self):r=requests.get(url=self.share_url)dytk_data=r.urlself.item_ids=re.search(r'video/(.+?)/',dytk_data).group(1)self.mid=re.search(r'mid=(.+?)&',dytk_data).group(1)self.u_code=re.search(r'u_code=(.+?)&',dytk_data).group(1)get_dytk()为了获取dytk参数defget_dytk(self):url=self.dytk_url.format(self.item_ids)r=requests.get(url=url,headers=self.headers,params={"region":"CN","mid":self.mid,"u_code":self.u_code,"titleType":"title","utm_source":"copy_link","utm_campaign":"client_share","utm_medium":"android","app":"aweme",})self.dytk=re.search(r'dytk:"(.+?)"}\);',r.text)get_infor()没有对返回的json进行处理defget_infor(self):r=requests.get(url=self.infor_url,headers=self.headers,params={"item_ids":self.item_ids,"dytk":self.dytk,})print(r.json())运行方法defrun(self):self.get_imu()self.get_dytk()self.get_infor()使用dys=DYSpider("https://v.douyin.com/Wf6Rsa/")dys.run()
爬虫目标通过微博话题获取发帖的信息与用户基本信息，数据可用于该话题的用户分析。爬取字段：账号id号、昵称、关注数、粉丝数、微博数、个性签名、账号类型、等级、简介、使用设备所用到模块：requests、lxml、re说明：无登录爬取微博用户数据，实现自动获取Cookie和Cookie失效自动更新，还有其他技巧，自己看。爬虫结构classWeiBoSpider():def__init__(self):#起始地址self.url="https://s.weibo.com/weibo?q=%23%E8%94%A1%E5%BE%90%E5%9D%A4%E7%82%B9%E8%AF%84%23&Refer=top"#cookieself.cookie=self.get_cookie()#获取所有数据defget_data(self):pass#通过id获取用户信息defget_user_data(self,user_id):pass#获取&更新Cookiedefget_cookie(self):passif__name__=='__main__':wbs=WeiBoSpider()wbs.get_data()获取所有数据defget_data(self):r=requests.get(url=self.url)html=etree.HTML(r.text)#接信息网页userdatas_list=html.xpath('//div[@id="pl_feedlist_index"]/div/div[@action-type="feed_list_item"]')#获取数据users_list=list()foruserdatainuserdatas_list:#昵称nick_name=userdata.xpath('.//div[@class="info"]/div[2]/a/@nick-name')#个人页user_id=re.search(r'/(\d*?)\?',userdata.xpath('.//div[@class="info"]/div[2]/a[1]/@href')[0]).group(1)#账号类型user_type=userdata.xpath('.//div[@class="info"]/div[2]/a[2]/@title')#使用设备equipment=userdata.xpath('.//p[@class="from"]/a[2]/text()')#组合数据data={"nick_name":nick_name,"user_id":user_id,"user_type":user_type,"equipment":equipment,#调用函数获取用户信息"user_data":self.get_user_data(user_id),}print(data)users_list.append(data)returnusers_list通过ID用户信息获取用户信息时，就对Cookie有着严格的要求，但是我们实现了自动化获取Cookie。defget_user_data(self,user_id="2803301701"):user_url="https://weibo.com/u/{}".format(user_id)headers={"Content-Type":"text/html;charset=utf-8","Host":"weibo.com",}r=requests.get(url=user_url,headers=headers,cookies=self.cookie)#判断cookie的有效性ifr.text=="":r=requests.get(url=user_url,headers=headers,cookies=self.get_cookie())#个人签名try:user_sign=re.search(r"简介：(.*?)<\\/span>",r.text).group(1).replace("\\t","").strip()exceptAttributeError:user_sign=""#关注concern=re.search(r'">(\d*?)<\\/strong><spanclass=\\"S_txt2\\">关注',r.text).group(1)#粉丝fans=re.search(r'">(\d*?)<\\/strong><spanclass=\\"S_txt2\\">粉丝<\\/span>',r.text).group(1)#微博数量weibo_count=re.search(r'">(\d*?)<\\/strong><spanclass=\\"S_txt2\\">微博',r.text).group(1)return{"user_sign":user_sign,"concern":concern,"fans":fans,"weibo_count":weibo_count}Cookie获取&更新在爬取微博用户数据遇到困难的朋友可以参考以下。defget_cookie(self):s=requests.session()#获取第一个Cookies.get("https://passport.weibo.com/visitor/visitor")#获取tid参数tid_=s.post(url="https://passport.weibo.com/visitor/genvisitor",data={"cb":"gen_callback","fp":{}})tid=re.search(r'{"tid":"(.+?)"',tid_.text).group(1)#返回需要的所有Cookier=s.get(url="https://passport.weibo.com/visitor/visitor",params={"a":"incarnate","t":tid,"cb":"cross_domain",})cookies=r.cookiestry:cookies["SRT"]exceptKeyError:print("获取cookie失败，真正重新获取")returnself.get_cookie()else:self.cookie=cookiesprint(self.cookie)returncookies运行结果暂没有翻页功能，看后面补充吧！！！
爬虫目标爬取房天下指定地区的所在小区、小区链接、小区地址、户型、总价、单价等使用的模块：requests、lxml、re、json爬虫结构以下结构可以很好的进行多线程或者协程的扩展。参数类classTool():#需要爬取的城市city="成都"#爬取页数page=2#城市列表获取所以城市的URL的方法在文末，这里只截取了一部分city_url={'成都':'https://cd.esf.fang.com','郴州':'https://chenzhou.esf.fang.com'}解析网页获取信息#获取房子的信息defget_fang_infor(url):#信息fang_list=list()#获取网页信息r=requests.get(url=url)r.encoding=r.apparent_encoding#这里的错误处理是在爬取第二页之后，URL需要获取一个参数try:href=re.search(r'//location.href="(.*?)";',r.text).group(1)exceptAttributeError:passelse:r=requests.get(url=href)r.encoding=r.apparent_encoding#解析网页html=etree.HTML(r.text)dl_html=html.xpath('//div[@class="shop_listshop_list_4"]/dl')fordlindl_html:#小区名称house=dl.xpath('.//p[@class="add_shop"]/a/@title')ifhouse==list():continue#小区链接house_url=url+dl.xpath('.//p[@class="add_shop"]/a/@href')[0]#小区地址house_add=dl.xpath('.//p[@class="add_shop"]/span/text()')#户型等去除前后符号house_types=[a.strip()foraindl.xpath('.//p[@class="tel_shop"]/text()')]#总价all_price=dl.xpath('.//dd[@class="price_right"]/span[1]/b/text()')[0]+dl.xpath('.//dd[@class="price_right"]/span/text()')[0]#单价price=dl.xpath('.//dd[@class="price_right"]/span[2]/text()')#组合数据infor={#小区名称"house":house[0],#小区地址"house_add":house_add[0],#小区链接"house_url":house_url,#户型"house_type":house_types[0],#大小"house_size":house_types[1][:-2],#楼层"floor":house_types[2],#方向"direction":house_types[3],#修建日期"xj_date":house_types[4],#总价"all_price":all_price,#单价"price":price[0][:-3]}fang_list.append(infor)returnfang_list获取所有链接返回一个所有链接的list()#获取页数连接defget_pages():url=Tool.city_url[Tool.city]pages_list=[url]fornuminrange(2,Tool.page+1):pages_list.append(url+"/house/i3{}/".format(num))returnpages_list主程序入口if__name__=='__main__':pages_list=get_pages()print(pages_list)#打印链接列表forpageinpages_list:print(page)#当前爬取的链接fang_list=get_fang_infor(page)print(fang_list)获取所有城市的URLdefget_city_url():url="https://esf.fang.com/esfcities.aspx"r=requests.get(url=url,timeout=5)r.encoding=r.apparent_encoding#匹配城市&urlcity_url_lists=json.loads(re.search(r"varcityJson=(.*?),];",r.text).group(1)+"]")#重新组合格式city_url_dic=dict()#{,...}foraincity_url_lists:city_url_dic={**city_url_dic,**{a.get("name"):"https:"+a.get("url")}}print(city_url_dic)
爬虫目标指定日期，爬取人民日报所有版面的新闻简要分析版面链接分析发现一共有20个版面，分别有20个链接，只是每个链接后的数字不同http://paper.people.com.cn/rmrb/html/2020-03/12/nbs.D110000renmrb_01.htm爬虫结构用到模块：requests、BeautifulSoup、gevent构造参数类需要下载其他日期，自行修改就好classTOOL():#年月yyyy-mm格式date="2020-03"#日期day="12"#新闻详情页text_url="http://paper.people.com.cn/rmrb/html/{}/{}/".format(date,day)#版面URL列表@classmethoddefget_urls(cls):urls=list()#链接列表fornuminrange(1,20):#10以下数字补0if1<=num<10:num="0"+str(num)urls.append("http://paper.people.com.cn/rmrb/html/{}/{}/nbs.D110000renmrb_{}.htm".format(cls.date,cls.day,num))returnurls获取新闻页链接获取一个版面的新闻链接defget_layout_url(url):"""返回值：[{"layout":layout,"href":href},{"layout":layout,"href":href},...]"""print(url)r=requests.get(url=url)r.encoding=r.apparent_encoding#解析网页获取版面名称、文章链接soup=BeautifulSoup(r.text,"html.parser")list_l_soup=soup.find(name="div",attrs={"class":"list_l"})#获取版面名layout=list_l_soup.find_next(name="div",attrs={"class":"l_t"}).string.strip()print(layout)ul_soup=list_l_soup.find_next(name="div",attrs={"id":"titleList"}).find_next(name="ul")li_soup=ul_soup.find_all(name="li")#返回值列表href_titles=list()forhref_title_taginli_soup:#获取hrefhref=TOOL.text_url+href_title_tag.find(name="a")["href"]print(href)#结果组合href_titles.append({"layout":layout,"href":href})returnhref_titles下载新闻并保存提前创建好rb/目录defdownload(href_titles):forhref_titleinhref_titles:#获取新闻页r=requests.get(url=href_title.get("href"))r.encoding=r.apparent_encoding#解析新闻页soup=BeautifulSoup(r.text,"html.parser")soup_text=soup.find(name="div",attrs={"class":"text_c"})#文章标题title=soup_text.h1.stringprint(title)#正文main_sc=soup_text.find_next(name="div",attrs={"id":"ozoom"})#正文里有一段script的注释，对结果有影响，需要替换掉main_sc.find_next(name="script",attrs={"language":"javascript"}).string=""main_text=main_sc.get_text().strip()#写入文件withopen("rb/"+title+".txt","w+",encoding="utf-8")asfile:file.write(main_text)主逻辑if__name__=='__main__':#通过for循环获取所有新闻链接，组成列表href_titles=[get_layout_url(url)forurlinTOOL.get_urls()]#添加协程队列并下载layout_url_list=[gevent.spawn(download,href_title)forhref_titleinhref_titles]gevent.joinall(layout_url_list)
爬虫目标下载必应搜索的背景壁纸使用模块：requests、gevent接口（get）：https://cn.bing.com/HPImageArchive.aspx主要参数"format":"js","idx":"0","n":"20",#壁纸数量，壁纸本身没几张，大了也没有爬虫主体获取壁纸连接返回包含壁纸URL和hash码的字典的列表defget_url_title():url="https://cn.bing.com/HPImageArchive.aspx"params={"format":"js","idx":"0","n":"20",#壁纸数量，壁纸本身没几张，大了也没有}r=requests.get(url=url,params=params).json()#存url和hsh的列表url_title_list=list()#解析jsonforimgdatainr.get("images"):hsh=imgdata.get("hsh")imgurl="https://cn.bing.com"+imgdata.get("url")#print(url,copyright)url_title_list.append({"url":imgurl,"hsh":hsh})returnurl_title_list下载并保存壁纸提前创建imgs/文件夹defdownload(url_title):#下载壁纸img_content=requests.get(url_title.get("url")).content#保存文件print(url_title.get("hsh"))withopen("imgs/"+url_title.get("hsh")+".jpg","wb+")asfile:file.write(img_content)主逻辑if__name__=='__main__':url_titles=get_url_title()#协程队列spawnList=[gevent.spawn(download,url_title)forurl_titleinurl_titles]#等待协程队列执行gevent.joinall(spawnList)