1 Star 0 Fork 2

ayibyb/基金信息爬取

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
index.js 6.33 KB
一键复制 编辑 原始数据 按行查看 历史
suheweiyang 提交于 2021-05-12 17:38 . init
const cheerio = require("cheerio");
const superagent = require("superagent");
const fs = require("fs");
// const nodeSchedule = require("node-schedule");
const sql = require('./sql/sql');
// 三年五星基金
var good_jj_list = 'https://fundzt.eastmoney.com/2017/newwxph/funds.js';
let jjList = 'http://fund.eastmoney.com/Data/Fund_JJJZ_Data.aspx?t=1&lx=1&letter=&gsid=&text=&sort=zdf,desc&page=1,10000&dt=1607933718705&atfc=&onlySale=1';
function getAllList() {
return new Promise((resolve, reject) => {
superagent.get(jjList, (err, res) => {
if (err) reject("request error");
eval(res.text);
let data = []
db.datas.forEach(item => {
let a = [];
item.forEach((it, index) => {
if ([12, 13, 14, 15, 16, 17, 18, 19, 20].indexOf(index) < 0) {
a.push(it)
}
});
var itemStr = a.join('')
if (itemStr.indexOf('开放申购') || itemStr.indexOf('开放赎回')) {
data.push(a)
}
});
data.forEach((d, index) => {
sql.update(d, index)
});
});
});
}
function getJijinHandle() {
sql.query().then((res) => {
async function insert() {
// for (var i = 0; i <res.length; i++) {
for (var i = res.length-1; i >-1; i--) {
await (function () {
return new Promise(function (resolve, rej) {
var item = res[i];
var jijinCode = item.id;
let jijinHandle = 'http://fundf10.eastmoney.com/FundArchivesDatas.aspx?type=jjcc&code=' + jijinCode + '&topline=10&year=&month=6&rt=0.5159413326499194';
console.log('start ', i, item.name);
superagent.get(jijinHandle, (err, res) => {
if (err) console.log("request error");
if (res && res.text) {
eval(res.text);
var $ = cheerio.load(res.text);
var allData = [];
$(".box").each(function (index, box) {
var $tds, number, gp_id, gp_name, jj_rate, quarter,year;
var text = $(box).find('h4.t .left').text();
var quarter_index = text.indexOf('季度');
quarter = text.substr(quarter_index - 1, 1);
var year_index = text.indexOf('');
year= text.substr(year_index - 4, 4);
if (index === 0) {
$(box).find('table tbody tr').each(function (index2, item) {
if (index2 > 10) {
return
}
$tds = $(item).find('td');
number = $tds.eq(0).text();
gp_id = $tds.eq(1).text();
gp_name = $tds.eq(2).find('a').text();
jj_rate = $tds.eq(6).text();
jj_gp_amount = $tds.eq(7).text();
jj_gp_money = $tds.eq(7).text();
var jj_gp_id = jijinCode + '_' + gp_id + '_'+year+'_' + quarter;
allData.push([number, jj_gp_id, gp_id, gp_name, jj_rate, jj_gp_amount, jj_gp_money, jijinCode,quarter,year])
})
} else {
$(box).find('table tbody tr').each(function (index2, item) {
if (index2 > 10) {
return
}
$tds = $(item).find('td');
number = $tds.eq(0).text();
gp_id = $tds.eq(1).text();
gp_name = $tds.eq(2).find('a').text();
jj_rate = $tds.eq(4).text();
jj_gp_amount = $tds.eq(5).text();
jj_gp_money = $tds.eq(6).text();
var jj_gp_id = jijinCode + '_' + gp_id + '_' + year+'_' +quarter;
allData.push([number, jj_gp_id, gp_id, gp_name, jj_rate, jj_gp_amount, jj_gp_money, jijinCode,quarter,year])
})
}
});
// console.log(allData)
async function fn() {
for (var i = 0; i < allData.length; i++) {
await (function () {
return new Promise(function (res1, rej) {
sql.insertJJDetail(allData[i], i, function (statu) {
console.log(JSON.stringify(statu));
res1()
})
})
}())
if (i === allData.length - 1) {
console.log('insert all success')
}
}
console.log('end ', i, item.name);
resolve();
}
fn();
}
})
})
}())
}
}
insert()
})
}
getJijinHandle();
// getAllList()
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
JavaScript
1
https://gitee.com/aibyb/fund-information-crawling.git
git@gitee.com:aibyb/fund-information-crawling.git
aibyb
fund-information-crawling
基金信息爬取
master

搜索帮助

23e8dbc6 1850385 7e0993f3 1850385