Csv YQL查询突然失败
我们正在利用Csv YQL查询突然失败,csv,yahoo,yql,Csv,Yahoo,Yql,我们正在利用YQL从CSV检索外部数据。最近,查询一直在全面返回null,我们无法找出原因 雅虎更新了他们的API吗 查询 http://query.yahooapis.com/v1/public/yql?q=select * from csv where url IN ('http://www.iso-ne.com/transform/csv/fiveminlmp/current?type=prelim') AND col1 IN ('4014') 商店 http://query.yahoo
YQL
从CSV检索外部数据。最近,查询一直在全面返回null
,我们无法找出原因
雅虎更新了他们的API吗
查询
http://query.yahooapis.com/v1/public/yql?q=select * from csv where url IN ('http://www.iso-ne.com/transform/csv/fiveminlmp/current?type=prelim') AND col1 IN ('4014')
商店
http://query.yahooapis.com/v1/public/yql?q=select * from yql.storage where name='store://ueZTPNC1PAVBw7lBz8FWNu'
<execute><![CDATA[
y.log("Version 4.1");
function getTimeQueryParam(){
var d = new Date();
var year = d.getFullYear();
var month = d.getMonth();
var day = d.getDate();
var hr = d.getHours();
var min = d.getMinutes();
var sec = d.getSeconds();
return '' + year + month + day + hr + min + sec;
}
//selects the current interval of LMP prices from ISO-NE
var newData = null;
newData = y.query("select * from csv where url IN ('http://www.iso-ne.com/transform/csv/fiveminlmp/current?type=prelim') AND col1 IN ('4014')");
newData = newData.results.row;
if(newData == null || newData == ''){
y.log("No New Data. Retry");
newData = y.query("select * from csv where url IN ('http://www.iso-ne.com/transform/csv/fiveminlmp/current?type=prelim') AND col1 IN ('4014')");
newData = newData.results.row;
}
y.log("Current time interval: " + newData.col2);
//selects the data stored in our YQL storage table for this tieline
var oldData = null;
oldData = y.query("select * from yql.storage where name='store://ueZTPNC1PAVBw7lBz8FWNu'");
var results = oldData.results.result;
y.log("results = oldData.results.result: " + results);
var resultsJSON = y.xmlToJson(results);
y.log("data JSON: " + y.jsToString(resultsJSON));
var data = resultsJSON.result.value;
var count = data.length;
//checks to make sure the current time interval has not been added to store
if(newData.col2 != data[data.length-1].col2){
//24 = MAX_NUMBER of intervals we want to keep
//delete oldest entries
y.log("Oldest time interval before delete: " + data[0].col2);
if(count > 24){
var deleteNum = count - 24;
for(d = 0; d < deleteNum; d++) data.shift();
}
y.log("Oldest time interval after delete: " +data[0].col2);
//adds current interval to store
data[data.length] = newData;
}
//creates JSONString from data Array, only text format can be stored in YQL storage
var txt = '[';
for(i = 0; i < data.length; i++){
txt = txt + '{"col0":"' + data[i].col0 + '", ';
txt = txt + '"col1":"' + data[i].col1 + '", ';
txt = txt + '"col2":"' + data[i].col2 + '", ';
txt = txt + '"col3":"' + data[i].col3 + '", ';
txt = txt + '"col4":"' + data[i].col4 + '", ';
txt = txt + '"col5":"' + data[i].col5 + '", ';
txt = txt + '"col6":"' + data[i].col6 + '"} ';
if(i+1 < data.length)txt = txt + ', ';
}
txt = txt + ']';
//updates the YQL Storage Table with new data set
var status = y.query("update yql.storage set value='" + txt + "'where name='" + update + "'");
y.log(status);
//pulls newly updated data for queries on this YQL table
var updatedData = y.query("select * from yql.storage where name='store://ueZTPNC1PAVBw7lBz8FWNu'");
response.object = updatedData.results.result;
]]></execute>
代码
http://query.yahooapis.com/v1/public/yql?q=select * from yql.storage where name='store://ueZTPNC1PAVBw7lBz8FWNu'
<execute><![CDATA[
y.log("Version 4.1");
function getTimeQueryParam(){
var d = new Date();
var year = d.getFullYear();
var month = d.getMonth();
var day = d.getDate();
var hr = d.getHours();
var min = d.getMinutes();
var sec = d.getSeconds();
return '' + year + month + day + hr + min + sec;
}
//selects the current interval of LMP prices from ISO-NE
var newData = null;
newData = y.query("select * from csv where url IN ('http://www.iso-ne.com/transform/csv/fiveminlmp/current?type=prelim') AND col1 IN ('4014')");
newData = newData.results.row;
if(newData == null || newData == ''){
y.log("No New Data. Retry");
newData = y.query("select * from csv where url IN ('http://www.iso-ne.com/transform/csv/fiveminlmp/current?type=prelim') AND col1 IN ('4014')");
newData = newData.results.row;
}
y.log("Current time interval: " + newData.col2);
//selects the data stored in our YQL storage table for this tieline
var oldData = null;
oldData = y.query("select * from yql.storage where name='store://ueZTPNC1PAVBw7lBz8FWNu'");
var results = oldData.results.result;
y.log("results = oldData.results.result: " + results);
var resultsJSON = y.xmlToJson(results);
y.log("data JSON: " + y.jsToString(resultsJSON));
var data = resultsJSON.result.value;
var count = data.length;
//checks to make sure the current time interval has not been added to store
if(newData.col2 != data[data.length-1].col2){
//24 = MAX_NUMBER of intervals we want to keep
//delete oldest entries
y.log("Oldest time interval before delete: " + data[0].col2);
if(count > 24){
var deleteNum = count - 24;
for(d = 0; d < deleteNum; d++) data.shift();
}
y.log("Oldest time interval after delete: " +data[0].col2);
//adds current interval to store
data[data.length] = newData;
}
//creates JSONString from data Array, only text format can be stored in YQL storage
var txt = '[';
for(i = 0; i < data.length; i++){
txt = txt + '{"col0":"' + data[i].col0 + '", ';
txt = txt + '"col1":"' + data[i].col1 + '", ';
txt = txt + '"col2":"' + data[i].col2 + '", ';
txt = txt + '"col3":"' + data[i].col3 + '", ';
txt = txt + '"col4":"' + data[i].col4 + '", ';
txt = txt + '"col5":"' + data[i].col5 + '", ';
txt = txt + '"col6":"' + data[i].col6 + '"} ';
if(i+1 < data.length)txt = txt + ', ';
}
txt = txt + ']';
//updates the YQL Storage Table with new data set
var status = y.query("update yql.storage set value='" + txt + "'where name='" + update + "'");
y.log(status);
//pulls newly updated data for queries on this YQL table
var updatedData = y.query("select * from yql.storage where name='store://ueZTPNC1PAVBw7lBz8FWNu'");
response.object = updatedData.results.result;
]]></execute>
24){
var deleteNum=计数-24;
对于(d=0;d
雅虎刚刚停止了对YQL
的支持。至少他们的html提取api
。您可以做的一件事是从社区开放数据表的一部分运行YQL函数
例如,用于提取html
,而不是:
select * from html
您可以使用:
select * from htmlstring
有没有发布或通知说支持被取消了?@PT_C是的,他们放弃了对html
的支持,但是htmlstring
仍然有效,因为雅虎的开发者社区支持在浏览器中运行查询,结果会说,Yahoo已经停止支持html
任何yql的web抓取替代方案?@Sinchan有一些选择,您可以通过搜索找到,否则我建议您构建自己的抓取器(服务器端)。每种语言/框架都有许多web scraper。如果您正在寻找实时性,我也会将其与ajax结合起来,以获得实时结果。如果您使用RubyonRails
。如果你提出问题,我可以写下答案。