python 爬取股市数据存入mysql

py3study

发布于 2020-01-14 21:05:52

3.1K0

发布于 2020-01-14 21:05:52

文章被收录于专栏：python3python3

1.利用casperjs 爬取新浪股市排行数据,生成数据文件

//获取新浪股票排行
var?casper?=?require('casper').create({
????waitTimeout:?10000,
????verbose:true,
????logLevel:?"debug",
????clientScripts:?["../jquery-3.1.1.min.js"],?#Casperjs的运行需要jquery
????pageSettings:?{
????????loadImages:??false,?
????????loadPlugins:?false,????
????userAgent:?'Mozilla/5.0?(Windows?NT?6.1;?rv:17.0)?Gecko/20100101?Firefox/17.0',
???},
});????
casper.start();??
//?2.?打开新浪股票热度排行页面??
casper.thenOpen('http://touzi.sina.com.cn/public/bhot');??
??
//?3.输出全市场用户关注1日变化幅度排行?
casper.then(function?getrank()?{??
??rank?=?casper.evaluate(function?()?{
????var?trlist?=?[];??
??????$("#allday1").find('tr:gt(0)').each(function(){
????????var?tdlist?=?[];
??????????$(this).find("td").each(function()?{
?????????????tdlist.push($(this).text().trim());
??????????});
??????????tdlist.push('\n');
??????????trlist.push(tdlist);
??????});?????????????????
??????return?trlist;
??});?
????var?filename?=?'data/allday1_sina.txt';
????var?fs?=?require('fs');
????fs.write(filename,rank);
});?
//?4.输出全市场用户关注5日变化幅度排行?
casper.then(function?getrank()?{??
??rank?=?casper.evaluate(function?()?{
????var?trlist?=?[];??
??????$("#allday5").find('tr:gt(0)').each(function(){
????????var?tdlist?=?[];
??????????$(this).find("td").each(function()?{
?????????????tdlist.push($(this).text().trim());
??????????});
??????????tdlist.push('\n');
??????????trlist.push(tdlist);
??????});?????????????????
??????return?trlist;
??});?
????var?filename?=?'data/allday5_sina.txt';
????var?fs?=?require('fs');
????fs.write(filename,rank);
});?
?
casper.run();

2.使用 python入库读取Casperjs生成的数据文件，写入mysql

#!/usr/bin/evn?python
#?-*-?coding:utf-8?-*-
import?MySQLdb
import?os,sys,subprocess

class?Spider():
???def?__init__(self,?filename,?js_filename,?table):
?????self.url?=?url
?????self.filename?=?filename?
?????self.table?=?table
???
???def?read_file_data(self):?????????
?????value_sets?=?[]?????????
?????try:?????????????
???????with?open(self.filename,?'rb')?as?data:
?????????count?=?0
?????????for?line?in?data:
???????????value?=?line.decode("gb2312").encode("utf-8").split()
???????????value_sets.append(value)?????????????????????
???????????count?+=?1?????????????
?????????return?value_sets,?count?????????
?????except?Exception,e:?????????????
????????print?Exception,":",e
????
???def?get_insert_sql(self):
??????"""
??????获取table字段list,返回插入数据sql语句
??????"""
?????try:
???????cursor?=?db.cursor()
???????cursor.execute("select?*?from?%s?limit?1"?%?self.table)
???????field_name_list?=?[each[0]?for?each?in?cursor.description]
???????del?field_name_list[0]??#去除自增id字段名
???????column_list?=?"("?+?",".join([field?for?field?in?field_name_list])?+?")"
???????values_format?=?"values("?+?("%s,"*len(field_name_list)).rstrip(',')?+?")"
???????insert_sql?=?"INSERT?INTO?%s"?%?self.table+column_list?+?values_format
???????return?insert_sql
?????except?Exception,?e:
???????print("Error:?%s"?%?e)
?????cursor.close()
?????db.close()
???????
???def?save(self):
?????value_sets,?count?=?self.get_file_data()
?????insert_sql?=?self.get_insert_sql()
?????if?not?(value_set?and?count):
???????print?"get?data?fom?file?failed"
?????if?not?insert_sql：
???????print?"get?insert_sql?failed"
?????try:
???????cursor?=?db.cursor()?
???????cursor.executemany(insert_sql,?value_sets)
???????db.commit()
???????print?(u"成功插入数据%d条"?%?count)
?????except?Exception,?e:
???????db.rollback()
???????print?Exception,?":",?e
???????print?(u"插入数据失败，数据回滚")
?????cursor.close()
?????db.close()

本文参与?腾讯云自媒体分享计划，分享自作者个人站点/博客。

原始发表：2019-06-26 ，如有侵权请联系 cloudcommunity@tencent.com 删除

python