1 Star 1 Fork 1

jiandanti/huobi_scrapy

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
btc_sql.py 5.22 KB
一键复制 编辑 原始数据 按行查看 历史
jiandanti 提交于 2020-04-19 14:36 . 爬虚拟货币信息并存入数据库
import configparser
import json
import os
import sys
import time
from locale import *
import pymysql
import requests
from bs4 import BeautifulSoup as bs
from apscheduler.schedulers.background import BackgroundScheduler
setlocale(LC_NUMERIC, 'zh_CN.utf-8')
log_mysql_host = None
log_mysql_port = None
log_mysql_user = None
log_mysql_pass = None
log_mysql_db = None
db = None
cursor = None
need_save = False
def read_cfg():
config = configparser.ConfigParser()
if len(sys.argv) < 2:
print("\033[0;31m请输入配置路径!!\033[0m")
exit()
cfgPath = sys.argv[1]
if not os.path.exists(cfgPath):
print("\033[0;31m路径 [" + cfgPath + "] 不存在!!\033[0m")
exit()
print("\033[0;35m配置路径为: " + cfgPath + " \033[0m")
config.read(cfgPath)
global log_mysql_db, log_mysql_pass, log_mysql_user, log_mysql_host, log_mysql_port
log_mysql_host = config.get("LogMySQL", "ip")
log_mysql_port = int(config.get("LogMySQL", "port"))
log_mysql_user = config.get("LogMySQL", "user")
log_mysql_pass = config.get("LogMySQL", "pass")
log_mysql_db = config.get("LogMySQL", "db")
print(" log mysql")
print("\033[0;34m ip = " + log_mysql_host + " \033[0m")
print("\033[0;34m port = " + str(log_mysql_port) + " \033[0m")
# print("\033[0;34m db = " + str(log_mysql_db) + " \033[0m")
pass
def connect_log_mysql():
global db, cursor
config = {
"host": log_mysql_host,
"user": log_mysql_user,
"password": log_mysql_pass,
"database": log_mysql_db,
"port": log_mysql_port,
"charset": "utf8"
}
db = pymysql.connect(**config)
cursor = db.cursor()
cursor.execute("SELECT VERSION()")
data = cursor.fetchone()
# print(data)
# print(type(data))
# print("Mysql connected success, version : " + str(data[0]))
# print("\033[0;32mMysql connected success, version : " + str("%s" % data) + " \033[0m")
def getCoin():
header = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
}
url = 'https://cn.investing.com/crypto/currencies'
url_huilv = 'https://webapi.huilv.cc/api/exchange?num=1&chiyouhuobi=USD&duihuanhuobi=CNY&callback=jisuanjieguo&_=1568021114461'
repos_data = requests.get(url_huilv, headers=header).content.decode("utf-8")
repos_data = repos_data.replace(')', '')
repos_data = repos_data.replace(';', '')
repos_data = repos_data.replace('jisuanjieguo(', '')
json_repos_data = repos_data.replace('(', '')
# print(json_repos_data)
dict_web_data = json.loads(json_repos_data)
# print(type(dict_web_data))
huilv = dict_web_data["dangqianhuilv"]
# print(type(huilv))
huilv_f = float(huilv)
# print(type(huilv_f))
# print(huilv_f)
# return huilv_f
web_data = requests.get(url, headers=header)
soup = bs(web_data.text, 'lxml')
sumdata = soup.select('#fullColumn > div > table > tbody') # 得到列表
# print(web_data.text)
# print(sumdata)
# pt = PrettyTable()
# pt._set_field_names(('排名 时间戳 日期 时间 名称 简称 价格(美元) 价格(人民币) 市值 24h成交量 交易份额 24小时涨跌 7日涨跌').split())
for onedata in sumdata[0].find_all('tr'):
data = onedata.find_all('td')
# print(data)
Rank = data[0].get_text() # 排名
Stime = time.time() # 时间戳
Tdate = time.strftime("%Y-%m-%d", time.localtime()) # 日期
Ttime = time.strftime("%H:%M:%S", time.localtime()) # 时间
Name = data[2].get_text() # 名称
Enname = data[3].get_text() # 英文名
PriceUSD = data[4].get_text() # 价格(美元)
# print(type(PriceUSD))
PriceCNY = atof(PriceUSD) * huilv_f # 价格(人民币) = 将千位分隔符的数字转化为常规数值 * (实时利率)
# print(PriceCNY)
MarketPrice = data[5].get_text() # 市值
Volumn24h = data[6].get_text() # 24小时成交量
TotalVol = data[7].get_text() # 交易份额
Change24h = data[8].get_text() # 24小时涨跌额
Change7d = data[9].get_text() # 7日涨跌额
cursor.execute("set global time_zone='+8:00';")
sql = "INSERT INTO btc_rate(Stime,Tdate,Ttime,Rank,Name,Enname,PriceUSD,PriceCNY,MarketPrice,Volumn24h,TotalVol,Change24h,Change7d)VALUES(%d,'%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')"
sql = sql % (
Stime, Tdate, Ttime, Rank, Name, Enname, PriceUSD, PriceCNY, MarketPrice, Volumn24h, TotalVol, Change24h,
Change7d)
try:
db.ping(True)
except Exception as e:
try:
cursor.close()
db.close()
except Exception as e:
print("close error")
connect_log_mysql()
cursor.execute(sql)
db.commit()
if __name__ == '__main__':
read_cfg()
connect_log_mysql()
try:
scheduler = BackgroundScheduler()
scheduler.add_job(getCoin, 'interval', seconds=5)
scheduler.start()
while True:
# print(time.time())
time.sleep(0)
except (KeyboardInterrupt, SystemExit):
print('Exit The Job!')
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Python
1
https://gitee.com/jiandanti/huobi_scrapy.git
git@gitee.com:jiandanti/huobi_scrapy.git
jiandanti
huobi_scrapy
huobi_scrapy
master

搜索帮助