爬虫：Python实现美女写真图库的抓取
要实现一个Python美女写真图库爬虫，首先需要分析目标网站的结构，找到图片的URL规律。然后使用requests库获取网页内容，使用BeautifulSoup库解析HTML，提取出图片URL。接着遍历图片URL列表，使用requests库下载图片并保存到本地。为了提高爬取效率，可以使用多线程或异步IO技术。在编写爬虫时，要注意遵守网站的robots.txt规则，尊重网站版权，合理设置爬取速度，避免对目标网站造成过大压力。
importrequestsfromlxmlimportetreeimportcsvfromtimeimportsleepimportosfromconcurrent.futuresimportThreadPoolExecutorheaders={'user-agent':'Mozilla/5.0(Macintosh;IntelMacOSX10_13_6)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.106Safari/537.36'}f=open('tulishe.csv',mode='w',encoding='gb18030')csvwriter=csv.writer(f)defdownload_one_page(url):page_text=requests.get(url=url,headers=headers).texttree=etree.HTML(page_text)post_list=tree.xpath('//div[@id="posts"]/div')print('------开始爬取第'+str(page_num)+'页------')fordivinpost_list:try:link=div.xpath('./div/a/@href')[0]#文章链接title=div.xpath('./div/a/@title')[0]#标题img=div.xpath('./div/a/img/@data-src')[0]#封面图img1=img.split('=')[1]#处理封面图url前缀img2=img1.split('&')[0]#处理封面图url后缀print('------开始下载---【'+title+'】---图片------')headers2={'User-Agent':'Mozilla/5.0(Macintosh;IntelMacOSX10_13_6)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.106Safari/537.36','Referer':link}#print(link,title)#开始请求详情页数据page2_text=requests.get(url=link,headers=headers).texttree2=etree.HTML(page2_text)item=tree2.xpath('/html/body/div[2]/div/div[2]/div/article/header/div/span[3]/a/text()')#分类article_tags=tree2.xpath('/html/body/div[2]/div/div[2]/div/article/div[3]/a')#标签组tags=[]forainarticle_tags:tag=a.xpath('./text()')tags.append(tag)#print(item,tags)all_pic_url=[]#详情页全部图片pic_urls=[]pic_list=tree2.xpath('//*[@id="gallery-2"]/div[@class="gallery-itemgallery-fancy-item"]')#详情页4图a=0fordivinpic_list:try:pic_url=div.xpath('./a/@href')[0]pic_urls.append(pic_url)all_pic_url.append(pic_url)#添加预览4图片到all_pic_url_#下载4预览图a=int(a)+1#print('--开始下载第',a,'预览图片--')img_data=requests.get(url=pic_url,headers=headers2).contentdir_name=pic_url.rsplit('.')[2].rsplit('/',1)[0]ifnotos.path.isdir(dir_name):try:original_umask=os.umask(0)os.makedirs(dir_name,mode=0o777)finally:os.umask(original_umask)withopen(pic_url.split(".",2)[2],'wb')asfp:fp.write(img_data)exceptExceptionase:continue#print('--',title,'-------',len(pic_urls),'张预览图--下载完成------')pic_list2=tree2.xpath('//*[@id="gallery-2"]/div[@class="gallery-itemgallery-blur-item"]')#详情页隐藏图pic_url3s=[]fordivinpic_list2:try:pic_url=div.xpath('./img/@src')[0]pic_url2=pic_url.split('=')[1]#解析隐藏链接图片pic_url3=pic_url2.split('&')[0]#解析隐藏链接图片pic_url3s.append(pic_url3)all_pic_url.append(pic_url3)#添加隐藏图片到all_pic_url_#下载隐藏图a=int(a)+1#print('--开始下载第',a,'隐藏图片--')img_data=requests.get(url=pic_url3,headers=headers2).contentdir_name=pic_url3.rsplit('.')[2].rsplit('/',1)[0]ifnotos.path.isdir(dir_name):try:original_umask=os.umask(0)os.makedirs(dir_name,mode=0o777)finally:os.umask(original_umask)withopen(pic_url3.split(".",2)[2],'wb')asfp:fp.write(img_data)exceptExceptionase:continue#print(title,'-------',len(pic_url3s),'张隐藏图--下载完成------')csvwriter.writerow([title,link,img2,item,tags,all_pic_url])#【标题；链接；封面图；分类；标签；详情页图片】保存到csvsleep(0.01)print(link,title,"爬取完毕！！！")#下载封面图img_data=requests.get(url=img2,headers=headers2).contentdir_name=img2.rsplit('.')[2].rsplit('/',1)[0]ifnotos.path.isdir(dir_name):try:original_umask=os.umask(0)os.makedirs(dir_name,mode=0o777)finally:os.umask(original_umask)withopen(img2.split(".",2)[2],'wb')asfp:fp.write(img_data)print('--',title,'封面图------下载完成------')exceptExceptionase:continueprint('第'+str(page_num)+'页，爬取完毕！！！')#exceptExceptionase:#continueif__name__=='__main__':withThreadPoolExecutor(100)ast:forpage_numinrange(1,842):#一共841页t.submit(download_one_page,f'http://www.tulishe.com/all/page/{page_num}')f.close()print("恭喜，全部爬取完毕！！！（文件为当前目录的tulishe.csv）")
