PythonÅÀȡèÑÛ¡¸µúÖеý¡¹È«²¿ÆÀÂÛ

# -*- coding:utf-8 -*-
import requests
import json
from datetime import datetime
import time
from tqdm import tqdm
from random import random

class MaoYan():
	"""docstring for ClassName"""
	def __init__(self, movie_id):
		print '*******MaoYan_spider******'
		print 'Author :     Awesome_Tang'
		print 'Date   :       2018-09-01'
		print 'Version:        Python2.7'
		print '**************************\n'
		self.movie_id = movie_id
		self.starttime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
		self.starturl = 'http://m.maoyan.com/mmdb/comments/movie/%s.json?_v_=yes&offset=0&startTime=%s'%(movie_id,self.starttime)
		self.headers = {'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'}
	
	def GetCommentNum(self):
		'''
		²éѯ×ÜÆÀÂÛÊý
		ÓÃÓÚ½¨Á¢Ñ­»·
		'''
		response = requests.get(self.starturl,headers = self.headers)
		text = response.json()
		num = text['total']
		print '>>>>²éѯʱ¼ä£º%s\n>>>>ÆÀÂÛÊýÁ¿£º%s'%(self.starttime,num)
		return num

	def FormatUrl(self,starttime):
		url = 'http://m.maoyan.com/mmdb/comments/movie/%s.json?_v_=yes&offset=30&startTime=%s'%(self.movie_id,starttime)
		return url

	def QueryComent(self,url):
		'''
		ÆÀÂÛÇëÇ󲿷Ö
		nickName:Óû§êdzÆ
		cityName:³ÇÊÐ
		content:ÆÀÂÛÄÚÈÝ
		score:Óû§ÆÀ·Ö??
		startTime:ÆÀÂÛʱ¼ä£¬Ã¿´ÎÈ¡×îÔçµÄʱ¼ä´«ÈëÏ´ÎÇëÇó
		'''
		try:	
			response = requests.get(url, headers = self.headers, timeout = 5)
			if response.status_code == 200:
				attrs = ''
				comments = response.json()['cmts']
				for index in range(15):
					try:
						nickName = comments[index]['nickName']
						cityName = comments[index]['cityName']
						content = comments[index]['content']
						score = comments[index]['score']
						startTime = comments[index]['startTime']
						param = '%s|%s|%s|%s|%s\n'%(startTime,nickName,cityName,score,content)
						attrs = attrs+param
					except KeyError as e:
						attrs = ''		
				return attrs ,startTime, True
			else:
				print '>>>>²éѯ¹ýÓÚƵ·±£¬ÇëÐÝÏ¢¼¸·ÖÖÓ????'
				return response.content.encode('utf-8'),'',False
		except BaseException as e:
			print '>>>>Çë¼ì²éÍøÂç...??\n'
			print e.message
			return e.message,'',False

	def SaveComent(self):
		'''
		±£´æÆÀÂÛµ½txtÎļþ
		Èç¹ûÇëÇó³É¹¦±£´æ£¬Ê§°Üsleep100Ãë
		tqdmÓÃÓÚʵÏÖ½ø¶ÈÌõ
		'''
		num = self.GetCommentNum()
		pages = num/15
		with open('comment.txt','a+') as f:
			for i in tqdm(range(pages)):
				if i == 0:
					starttime = self.starttime
					url = self.FormatUrl(starttime)
					attrs,starttime,IsOk = self.QueryComent(url)
				else:
					url = self.FormatUrl(starttime)
					attrs,starttime,IsOk = self.QueryComent(url)
				if IsOk:
					f.write(attrs.encode('utf-8'))
				else:
					while True:
						time.sleep(100)
						attrs,starttime,IsOk = self.QueryComent(url)
						if IsOk:
							f.write(attrs.encode('utf-8'))
							break
						else:
							continue
			print '>>>>ÆÀÂÛ±£´æÍê±Ï...'



if __name__ == '__main__':
	p = MaoYan('341737')
	p.SaveComent()