GitHub³¬¼¶»ð£¡ÈÎÒâÅÀÈ¡£¬³¬È«¿ªÔ´ÅÀ³æ¹¤¾ßÏä

/*
* Ìáʾ£º¸ÃÐдúÂë¹ý³¤£¬ÏµÍ³×Ô¶¯×¢ÊͲ»½øÐиßÁÁ¡£Ò»¼ü¸´ÖÆ»áÒƳýϵͳעÊÍ 
* import?json??import?random??import?time??import?sys??import?os??import?requests??import?numpy?as?np??import?math??from?lxml?import?etree??from?pyquery?import?PyQuery?as?pq??from?selenium?import?webdriver??from?selenium.webdriver?import?ChromeOptions??from?selenium.webdriver.common.by?import?By??from?selenium.webdriver.support?import?expected_conditions?as?EC??from?selenium.webdriver.support.wait?import?WebDriverWait??from?selenium.webdriver?import?ChromeOptions,?ActionChains??from?tkinter.filedialog?import?askdirectory??from?tqdm?import?trange????def?ease_out_quad(x):??return?1?-?(1?-?x)?*?(1?-?x)??def?ease_out_quart(x):??return?1?-?pow(1?-?x,?4)??def?ease_out_expo(x):??if?x?==?1:??return?1??else:??return?1?-?pow(2,?-10?*?x)??def?get_tracks(distance,?seconds,?ease_func):??tracks?=?[0]??offsets?=?[0]??for?t?in?np.arange(0.0,?seconds,?0.1):??ease?=?globals()[ease_func]??offset?=?round(ease(t?/?seconds)?*?distance)??tracks.append(offset?-?offsets[-1])??offsets.append(offset)??return?offsets,?tracks??def?drag_and_drop(browser,?offset=26.5):??knob?=?browser.find_element_by_id('nc_1_n1z')??offsets,?tracks?=?get_tracks(offset,?12,?'ease_out_expo')??ActionChains(browser).click_and_hold(knob).perform()??for?x?in?tracks:??ActionChains(browser).move_by_offset(x,?0).perform()??ActionChains(browser).pause(0.5).release().perform()??def?gen_session(cookie):??session?=?requests.session()??cookie_dict?=?{}??list?=?cookie.split(';')??for?i?in?list:??try:??cookie_dict[i.split('=')[0]]?=?i.split('=')[1]??except?IndexError:??cookie_dict['']?=?i??requests.utils.add_dict_to_cookiejar(session.cookies,?cookie_dict)??return?session??class?TaobaoSpider(object):??def?__init__(self,?cookies_list):??self.path?=?askdirectory(title='Ñ¡ÔñÐÅÏ¢±£´æÎļþ¼Ð')??if?str(self.path)?==?"":??sys.exit(1)??self.headers?=?{??'User-Agent':?'Mozilla/5.0?(Macintosh;?Intel?Mac?OS?X?10_14_3)?AppleWebKit/537.36?(KHTML,?like?Gecko)?Chrome/73.0.3683.86?Safari/537.36',??}??option?=?ChromeOptions()??option.add_experimental_option('excludeSwitches',?['enable-automation'])??option.add_experimental_option("prefs",?{"profile.managed_default_content_settings.images":?2})?#?²»¼ÓÔØͼƬ,¼Ó¿ì·ÃÎÊËÙ¶È??option.add_argument('--headless')??self.driver?=?webdriver.Chrome(options=option)??self.driver.get('https://i.taobao.com/my_taobao.htm')??for?i?in?cookies_list:??self.driver.add_cookie(cookie_dict=i)??self.driver.get('https://i.taobao.com/my_taobao.htm')??self.wait?=?WebDriverWait(self.driver,?20)?#?³¬Ê±Ê±³¤Îª10s??#?Ä£ÄâÏòÏ»¬¶¯ä¯ÀÀ??def?swipe_down(self,?second):??for?i?in?range(int(second?/?0.1)):??#?¸ù¾ÝiµÄÖµ£¬Ä£ÄâÉÏÏ»¬¶¯??if?(i?%?2?==?0):??js?=?"var?q=document.documentElement.scrollTop="?+?str(300?+?400?*?i)??else:??js?=?"var?q=document.documentElement.scrollTop="?+?str(200?*?i)??self.driver.execute_script(js)??time.sleep(0.1)??js?=?"var?q=document.documentElement.scrollTop=100000"??self.driver.execute_script(js)??time.sleep(0.1)??#?ÅÀÈ¡ÌÔ±¦?ÎÒÒÑÂòµ½µÄ±¦±´ÉÌÆ·Êý¾Ý,?pn?¶¨ÒåÅÀÈ¡¶àÉÙÒ³Êý¾Ý??def?crawl_good_buy_data(self,?pn=3):??#?¶ÔÎÒÒÑÂòµ½µÄ±¦±´ÉÌÆ·Êý¾Ý½øÐÐÅÀ³æ??self.driver.get("https://buyertrade.taobao.com/trade/itemlist/list_bought_items.htm")??#?±éÀúËùÓÐÒ³Êý????for?page?in?trange(1,?pn):??data_list?=?[]??#?µÈ´ý¸ÃÒ³ÃæÈ«²¿ÒÑÂòµ½µÄ±¦±´ÉÌÆ·Êý¾Ý¼ÓÔØÍê±Ï??good_total?=?self.wait.until(??EC.presence_of_element_located((By.CSS_SELECTOR,?'#tp-bought-root?>?div.js-order-container')))??#?»ñÈ¡±¾Ò³ÃæÔ´´úÂë??html?=?self.driver.page_source??#?pqÄ£¿é½âÎöÍøÒ³Ô´´úÂë??doc?=?pq(html)??#?#?´æ´¢¸ÃÒ³ÒѾ­Âòµ½µÄ±¦±´Êý¾Ý??good_items?=?doc('#tp-bought-root?.js-order-container').items()??#?±éÀú¸ÃÒ³µÄËùÓᦱ´??for?item?in?good_items:??#?ÉÌÆ·¹ºÂòʱ¼ä¡¢¶©µ¥ºÅ??good_time_and_id?=?item.find('.bought-wrapper-mod__head-info-cell___29cDO').text().replace('\n',?"").replace('\r',?"")??#?É̼ÒÃû³Æ??#?good_merchant?=?item.find('.seller-mod__container___1w0Cx').text().replace('\n',?"").replace('\r',?"")??good_merchant?=?item.find('.bought-wrapper-mod__seller-container___3dAK3').text().replace('\n',?"").replace('\r',?"")??#?ÉÌÆ·Ãû³Æ??#?good_name?=?item.find('.sol-mod__no-br___1PwLO').text().replace('\n',?"").replace('\r',?"")??good_name?=?item.find('.sol-mod__no-br___3Ev-2').text().replace('\n',?"").replace('\r',?"")??#?ÉÌÆ·¼Û¸ñ??good_price?=?item.find('.price-mod__price___cYafX').text().replace('\n',?"").replace('\r',?"")??#?Ö»ÁгöÉÌÆ·¹ºÂòʱ¼ä¡¢¶©µ¥ºÅ¡¢É̼ÒÃû³Æ¡¢ÉÌÆ·Ãû³Æ??#?ÆäÓàµÄÇë×Ô¼ºÊµ¼ù»ñÈ¡??data_list.append(good_time_and_id)??data_list.append(good_merchant)??data_list.append(good_name)??data_list.append(good_price)??#print(good_time_and_id,?good_merchant,?good_name)??#file_path?=?os.path.join(os.path.dirname(__file__)?+?'/user_orders.json')??#?file_path?=?"../Spiders/taobao/user_orders.json"??json_str?=?json.dumps(data_list)??with?open(self.path?+?os.sep?+?'user_orders.json',?'a')?as?f:??f.write(json_str)??#?print('\n\n')??#?´ó²¿·ÖÈ˱»¼ì²âΪ»úÆ÷È˾ÍÊÇÒòΪ½øÒ»²½Ä£ÄâÈ˹¤²Ù×÷??#?Ä£ÄâÈ˹¤ÏòÏÂä¯ÀÀÉÌÆ·£¬¼´½øÐÐÄ£ÄâÏ»¬²Ù×÷£¬·ÀÖ¹±»Ê¶±ð³öÊÇ»úÆ÷ÈË??#?Ëæ»ú»¬¶¯ÑÓʱʱ¼ä??swipe_time?=?random.randint(1,?3)??self.swipe_down(swipe_time)??#?µÈ´ýÏÂÒ»Ò³°´Å¥?³öÏÖ??good_total?=?self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,?'.pagination-next')))??good_total.click()??time.sleep(2)??#?while?1:??#?time.sleep(0.2)??#?try:??#?good_total?=?self.driver.find_element_by_xpath('//li[@title="ÏÂÒ»Ò³"]')??#?break??#?except:??#?continue??#?#?µã»÷ÏÂÒ»Ò³°´Å¥??#?while?1:??#?time.sleep(2)??#?try:??#?good_total.click()??#?break??#?except?Exception:??#?pass??#?Êղر¦±´?´«ÈëÅÀ¼¸Ò³?ĬÈÏÈýÒ³?https://shoucang.taobao.com/nodejs/item_collect_chunk.htm?ifAllTag=0&tab=0&tagId=&categoryCount=0&type=0&tagName=&categoryName=&needNav=false&startRow=60??def?get_choucang_item(self,?page=3):??url?=?'https://shoucang.taobao.com/nodejs/item_collect_chunk.htm?ifAllTag=0&tab=0&tagId=&categoryCount=0&type=0&tagName=&categoryName=&needNav=false&startRow={}'??pn?=?0??json_list?=?[]??for?i?in?trange(page):??self.driver.get(url.format(pn))??pn?+=?30??html_str?=?self.driver.page_source??if?html_str?==?'':??break??if?'µÇ¼'?in?html_str:??raise?Exception('µÇ¼')??obj_list?=?etree.HTML(html_str).xpath('//li')??for?obj?in?obj_list:??item?=?{}??item['title']?=?''.join([i.strip()?for?i?in?obj.xpath('./div[@class="img-item-title"]//text()')])??item['url']?=?''.join([i.strip()?for?i?in?obj.xpath('./div[@class="img-item-title"]/a/@href')])??item['price']?=?''.join([i.strip()?for?i?in?obj.xpath('./div[@class="price-container"]//text()')])??if?item['price']?==?'':??item['price']?=?'ʧЧ'??json_list.append(item)??#?file_path?=?os.path.join(os.path.dirname(__file__)?+?'/shoucang_item.json')??json_str?=?json.dumps(json_list)??with?open(self.path?+?os.sep?+?'shoucang_item.json',?'w')?as?f:??f.write(json_str)??#?ä¯ÀÀ×ã¼£?´«ÈëÅÀ¼¸Ò³?ĬÈÏÈýÒ³?https://shoucang.taobao.com/nodejs/item_collect_chunk.htm?ifAllTag=0&tab=0&tagId=&categoryCount=0&type=0&tagName=&categoryName=&needNav=false&startRow=60??def?get_footmark_item(self,?page=3):??url?=?'https://www.taobao.com/markets/footmark/tbfoot'??self.driver.get(url)??pn?=?0??item_num?=?0??json_list?=?[]??for?i?in?trange(page):??html_str?=?self.driver.page_source??obj_list?=?etree.HTML(html_str).xpath('//div[@class="item-list?J_redsList"]/div')[item_num:]??for?obj?in?obj_list:??item_num?+=?1??item?=?{}??item['date']?=?''.join([i.strip()?for?i?in?obj.xpath('./@data-date')])??item['url']?=?''.join([i.strip()?for?i?in?obj.xpath('./a/@href')])??item['name']?=?''.join([i.strip()?for?i?in?obj.xpath('.//div[@class="title"]//text()')])??item['price']?=?''.join([i.strip()?for?i?in?obj.xpath('.//div[@class="price-box"]//text()')])??json_list.append(item)??self.driver.execute_script('window.scrollTo(0,1000000)')??#?file_path?=?os.path.join(os.path.dirname(__file__)?+?'/footmark_item.json')??json_str?=?json.dumps(json_list)??with?open(self.path?+?os.sep?+?'footmark_item.json',?'w')?as?f:??f.write(json_str)??#?µØÖ·??def?get_addr(self):??url?=?'https://member1.taobao.com/member/fresh/deliver_address.htm'??self.driver.get(url)??html_str?=?self.driver.page_source??obj_list?=?etree.HTML(html_str).xpath('//tbody[@class="next-table-body"]/tr')??data_list?=?[]??for?obj?in?obj_list:??item?=?{}??item['name']?=?obj.xpath('.//td[1]//text()')??item['area']?=?obj.xpath('.//td[2]//text()')??item['detail_area']?=?obj.xpath('.//td[3]//text()')??item['youbian']?=?obj.xpath('.//td[4]//text()')??item['mobile']?=?obj.xpath('.//td[5]//text()')??data_list.append(item)??#?file_path?=?os.path.join(os.path.dirname(__file__)?+?'/addr.json')??json_str?=?json.dumps(data_list)??with?open(self.path?+?os.sep?+?'address.json',?'w')?as?f:??f.write(json_str)????if?__name__?==?'__main__':??#?pass??cookie_list?=?json.loads(open('taobao_cookies.json',?'r').read())??t?=?TaobaoSpider(cookie_list)??t.get_orders()??#?t.crawl_good_buy_data()??#?t.get_addr()??#?t.get_choucang_item()??#?t.get_footmark_item()
*/