请问怎么嵌入代理ip 目标是实现多个ip 爬取商品的多页评论 麻烦大神对代码优化 谢谢

阿☆輝

1. #商品链接 http://detail.tmall.com/item.htm?id=41464129793
   
2. #爬取商品多页评论：
   
3. import requests
   
4. import re
   
5. import pandas as pd
   
6. from pandas import DataFrame
   
7. import time
   
8. 
   
9. datapj = DataFrame(columns=[])
   
10. for i in range(1,100):   
    
11.    i = str(i)  
    
12.    url='http://rate.tmall.com/list_detail_rate.htm?itemId=41464129793&sellerId=1652490016&currentPage=' + i
    
13.    web = requests.get(url)
    
14.    json = re.findall('rateList":(.*?),"searchinfo',web.text)[0]
    
15.    table = pd.read_json(json)
    
16.    datapj = pd.concat([datapj,table],axis=0,ignore_index=True)
    
17.    time.sleep(10)
    
18.            
    
19. datapj.to_excel('datapj.xls')

复制代码

阿☆輝

1. # 1. 爬取商品多页评论：
   
2. #商品链接: https://detail.tmall.com/item.htm?id=555502261542
   
3. #url: https://rate.tmall.com/list_detail_rate.htm?itemId=555502261542&sellerId=813836783&currentPage=1
   
4. #user-agent:Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36
   
5. 
   
6. import requests
   
7. import urllib.request
   
8. import re
   
9. import pandas as pd
   
10. from pandas import DataFrame
    
11. import time
    
12. import random
    
13. 
    
14. tlist = [6,8,11,15,16,18,22]
    
15. #iplist = ['119.29.18.239:8888', '219.138.58.74:3128', '114.228.8.128:8118','60.177.225.111:808','180.156.95.80:8118']  #类型：https
    
16. iplist = ['221.225.186.63:3128', '219.244.186.30:3128', '122.72.18.34:80','58.220.95.107:8080','116.31.75.100:3128']
    
17. datapj = DataFrame(columns=[])
    
18. for i in range(1,100):   
    
19.    i = str(i)  
    
20.    url='https://rate.tmall.com/list_detail_rate.htm?itemId=555502261542&sellerId=813836783&currentPage=' + i
    
21.    proxy = urllib.request.ProxyHandler({'http':random.choice(iplist)})    #随机选择代理ip
    
22.    opener = urllib.request.build_opener(proxy)      #创建opener模块
    
23.    opener.addheaders = [('user-agent','Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36')]  #添加headers
    
24.    urllib.request.install_opener(opener)     #安装opener
    
25.    web = requests.get(url)
    
26.    json = re.findall('rateList":(.*?),"searchinfo',web.text)[0]
    
27.    table = pd.read_json(json)
    
28.    datapj = pd.concat([datapj,table],axis=0,ignore_index=True)
    
29.    time.sleep(random.choice(tlist))
    
30. 
    
31. datapj.to_excel('datapj.xls')

复制代码

这是我对本贴问题的完善  我写的这个代码  只能爬取几十页就gameover   请问如何进一步优化修改？  我想爬取所有 99 页的评论  大神请赐教？？？ 所有鱼币奉上