C# SQL程序集WebResponse和字符串解析非常慢
因此，我很快就学会了C#（继承了这个问题的完整noob）的方法；我已经编写了下面的代码，它调用了一个返回JSON的web服务，该JSON并不总是格式良好的。这里的任务是获取JSON字符串并将其分解为数组段，这些数组段被插入到SQL表中进行进一步的解析和测试。即，如果返回字符串类似于C# SQL程序集WebResponse和字符串解析非常慢,c#,sql-server,web-services,httpwebrequest,sqlclr,C#,Sql Server,Web Services,Httpwebrequest,Sqlclr,因此，我很快就学会了C#（继承了这个问题的完整noob）的方法；我已经编写了下面的代码，它调用了一个返回JSON的web服务，该JSON并不总是格式良好的。这里的任务是获取JSON字符串并将其分解为数组段，这些数组段被插入到SQL表中进行进一步的解析和测试。即，如果返回字符串类似于 {1234:{5678:{1:{"key":"val","key":"val"},{2:{"key":"val","key":"val"}}}} 那么这些行将是： {1234} {5678} {1:{"key
{1234:{5678:{1:{"key":"val","key":"val"},{2:{"key":"val","key":"val"}}}}
{1234}
{5678}
{1:{"key":"val","key":"val"}
{2:{"key":"val","key":"val"}
public partial class UserDefinedFunctions
{
[Microsoft.SqlServer.Server.SqlFunction(DataAccess =
DataAccessKind.Read)]
public static SqlString TestParse(SqlString uri, SqlString username, SqlString passwd, SqlString postdata)
{
//-----
// The SqlPipe is how we send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
SqlString document;
try
{
// Set up the request, including authentication
WebRequest req = WebRequest.Create(Convert.ToString(uri));
if (Convert.ToString(username) != null & Convert.ToString(username) != "")
{
req.Credentials = new NetworkCredential(
Convert.ToString(username),
Convert.ToString(passwd));
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
//SqlContext.Pipe.Send("...get the data");
using (StreamReader rdr = new StreamReader(dataStream))
{
document = (SqlString)rdr.ReadToEnd();
rdr.Close();
//-----
string connectionString = null;
string sql = null;
connectionString = "Data source= 192.168.0.5; Database=Administration;User Id=Foo;Password=Blah; Trusted_Connection=True;";
using (SqlConnection cnn = new SqlConnection(connectionString))
{
sql = "INSERT INTO JSON_DATA (JSONROW) VALUES(@data)";
cnn.Open();
using (SqlCommand cmd = new SqlCommand(sql, cnn))
{
String payload = "";
String nestpayload = "";
int nests = 0;
String json = document.ToString();
/*first lets do some housekeeping on our payload; double closing curly braces need to be escaped (with curly braces!) in order to keep them in the string.*/
json = json.Replace("\\", "");
int i = json.Length;
//return new SqlString(json);
while (i > 1)
{
/*find the first closing "}" in the string and then check to see if there are more than one.
We need to read the data up to each closing brace, pull off that substring and process it for each iteration until the string is gone.*/
int closingbrace = json.IndexOf("}"); //First closing brace
int nextbrace = Math.Max(0, json.IndexOf("{", closingbrace)); //Next opening brace
String ChkVal = json.Substring(closingbrace + 1, Math.Max(1, nextbrace - closingbrace)); //+1 to ignore the 1st closing brace
int checks = Math.Max(0, ChkVal.Length) - Math.Max(0, ChkVal.Replace("}", "").Length);
payload = json.Substring(0, Math.Max(0, (json.IndexOf("}") + 1)));
/*Remove the payload from the string*/
json = json.Substring(payload.Length + 1);
/*"nests" is how many nested levels excluding the opening brace for the closing brace we found.*/
nests = (payload.Length - payload.Replace("{", "").Length);
/*If we have more then one nest level check to see if any of them go with the payload*/
if (nests > 1)
{
/*Break out the nested section and remove it from the payload.*/
nestpayload = payload.Substring(0, payload.LastIndexOf("{"));
payload = payload.Substring(payload.LastIndexOf("{"), payload.Length - payload.LastIndexOf("{"));
while (nests > 1)
{
if (checks > 0) //# of right braces in payload equals number of left-side nests go with the payload
{
// payload = nestpayload.Substring(Math.Max(0, nestpayload.LastIndexOf("{")), Math.Max(0, nestpayload.Length) - Math.Max(0, (nestpayload.LastIndexOf("{")))) + payload;//The second Math.Max defaults to 1; if we got here there is at minimum one "{" character in the substring
payload = nestpayload.Substring(nestpayload.LastIndexOf("{")) + payload;
nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0, nestpayload.LastIndexOf("{"))));
checks--;
nests--;
}
else
{
/*If we got here there are no more pieces of the nested data to append to the payload.
We use an array and string.split to keep the nest ordering correct.*/
string[] OrderedNest = nestpayload.Split('{');
for (int s = 0; s < OrderedNest.Length; s++)
{
if (OrderedNest[s] != "")
{
cmd.Parameters.AddWithValue("@data", "{" + OrderedNest[s].Replace(":", "}"));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
}
//cmd.Parameters.AddWithValue("@data", nestpayload.Substring(Math.Max(0,nestpayload.LastIndexOf("{"))).Replace(":","}"));
//cmd.Parameters.AddWithValue("@data", OrderedNest[1].Replace(":","}")+OrderedNest[2]);
// cmd.ExecuteNonQuery();
//cmd.Parameters.Clear();
//nests = Math.Max(0, nests - 1);
nests = 0;
//nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0,nestpayload.LastIndexOf("{"))));
}
}
}
/*At the very end payload will be a single "}"; check for this and discard the last row*/
if (payload != "}")
{
cmd.Parameters.AddWithValue("@data", new SqlChars(payload));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
/*Get the new string length*/
i = json.Length;
payload = "";
}
}
}
//-----
/* }
catch (Exception e)
{
return e.ToString();
}*/
}
// Close up everything...
dataStream.Close();
}
resp.Close();
// .. and return the output to the caller.
}//end using
return ("Finished");
}
catch (WebException e)
{
throw e;
}
}
}
公共部分类UserDefinedFunctions
{
[Microsoft.SqlServer.Server.SqlFunction（数据访问=
DataAccessKind.Read）]
公共静态SqlString TestParse（SqlString uri、SqlString用户名、SqlString密码、SqlString postdata）
{
//-----
//SqlPipe是我们将数据发送回调用方的方式
SqlPipe=SqlContext.pipe；
SqlString文档；
尝试
{
//设置请求，包括身份验证
WebRequest req=WebRequest.Create（Convert.ToString（uri））；
if（Convert.ToString（用户名）！=null&Convert.ToString（用户名）！=“”）
{
请求凭据=新网络凭据(
Convert.ToString（用户名），
Convert.ToString（passwd））；
}
（（HttpWebRequest）req.UserAgent=“SQL Server上的CLR web客户端”；
//触发请求并检索响应。
使用（WebResponse resp=req.GetResponse（））
{
使用（Stream dataStream=resp.GetResponseStream（））
{
//SqlContext.Pipe.Send（“…获取数据”）；
使用（StreamReader rdr=新的StreamReader（数据流））
{
document=（SqlString）rdr.ReadToEnd（）；
rdr.Close（）；
//-----
字符串连接字符串=null；
字符串sql=null；
connectionString=“数据源=192.168.0.5；数据库=管理；用户Id=Foo；密码=Blah；可信的_连接=True；”；
使用（SqlConnection cnn=newsqlconnection（connectionString））
{
sql=“插入JSON_数据（JSONROW）值（@DATA）”；
cnn.Open（）；
使用（SqlCommand cmd=newsqlcommand（sql，cnn））
{
字符串有效载荷=”；
字符串nestpayload=“”；
int nests=0；
字符串json=document.ToString（）；
/*首先，让我们对有效负载进行一些整理；需要转义双闭合花括号（使用花括号！），以便将它们保留在字符串中*/
json=json.Replace（“\\”，“”）；
inti=json.Length；
//返回新的SqlString（json）；
而（i>1）
{
/*在字符串中找到第一个结束符“}”，然后检查是否有多个结束符。
我们需要将数据读取到每个右大括号，取出子字符串，并在每次迭代中处理它，直到字符串消失*/
int closingbrace=json.IndexOf（“}”）；//第一个右大括号
int nextbrace=Math.Max（0，json.IndexOf（“{”，closingbrace））；//下一个大括号
字符串ChkVal=json.Substring（closingbrace+1，Math.Max（1，nextbrace-closingbrace））；//+1忽略第一个右大括号
int checks=Math.Max（0，ChkVal.Length）-Math.Max（0，ChkVal.Replace（“}”，”）.Length）；
payload=json.Substring（0，Math.Max（0，（json.IndexOf（“}”）+1））；
/*从管柱上卸下有效负载*/
json=json.Substring（payload.Length+1）；
/*“嵌套”是指我们找到的不包括左大括号和右大括号的嵌套层数*/
nests=（payload.Length-payload.Replace（“{”，”）.Length）；
/*如果我们有一个以上的嵌套级别，请检查它们是否与有效负载匹配*/
如果（嵌套>1）
{
/*拆开嵌套部分并将其从有效负载上拆下*/
nestpayload=payload.Substring（0，payload.LastIndexOf（“{”）；
payload=payload.Substring（payload.LastIndexOf（“{”）、payload.Length-payload.LastIndexOf（“{”）；
而（嵌套>1）
{
如果（检查值>0）/#有效载荷中的右支架数量等于有效载荷中的左侧嵌套数量
{
//payload=nestpayload.Substring（Math.Max（0，nestpayload.LastIndexOf（“{”）），Math.Max（0，nestpayload.Length）-Math.Max（0，nestpayload.LastIndexOf（“{”））+payload；//第二个Math.Max默认为1；如果我们到这里，子字符串中至少有一个“{”字符
有效载荷=nestpayload.Substring（nestpayload.LastIndexOf（“{”））+有效载荷；
nestpayload=nestpayload.Substring（0，Math.Max（0，Math.Max（0，nestpayload.LastIndexOf（“{”）））；
DECLARE @JSON NVARCHAR(MAX);
SELECT @JSON = [content]
FROM SQL#.INET_GetWebPages(@uri, .....);
INSERT INTO dbo.JSON_DATA (JSONROW)
SELECT [column_name]
FROM dbo.MyBrokenJsonFixerUpper(@JSON);
using System;
using System.Data;
using System.Data.SqlClient;
using System.Data.SqlTypes;
using Microsoft.SqlServer.Server;
using System.Collections;
using System.Globalization;
// Other things we need for WebRequest
using System.Net;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;
public partial class StoredProcedures
{
[Microsoft.SqlServer.Server.SqlProcedure]
public static void ApiParser(SqlString uri, SqlString user, SqlString pwd, SqlString postd)
{
// Create an SqlPipe to send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
//Make sure we have a url to process
if (uri.IsNull || uri.Value.Trim() == string.Empty)
{
pipe.Send("uri cannot be empty");
return;
}
try
{
//Create our datatable and get the table structure from the database
DataTable table = new DataTable();
string connectionString = null;
//connectionString = "Data source= 192.168.0.5; Database=Administration; Trusted_Connection=True;";
connectionString = "Data Source=(localdb)\\ProjectsV12;Integrated Security=True;Connect Timeout=30;Encrypt=False;TrustServerCertificate=False;ApplicationIntent=ReadWrite;MultiSubnetFailover=False";
using (SqlConnection gts = new SqlConnection(connectionString))
{
gts.Open();
using (SqlDataAdapter adapter = new SqlDataAdapter("SELECT TOP 0 * FROM sp_WebSvcs.dbo.JSON_DATA", gts))
{
adapter.Fill(table);
}
}
// Send a message string back to the client.
pipe.Send("Beginning Api Call...");
String json = "";
// Set up the request, including authentication
WebRequest req = HttpWebRequest.Create(uri.Value);
if (!user.IsNull & user.Value != "")
{
req.Credentials = new NetworkCredential(user.Value, pwd.Value);
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
using (StreamReader rdr = new StreamReader(dataStream))
{
json = (String)rdr.ReadToEnd();
rdr.Close();
}
// Close up everything...
dataStream.Close();
}
resp.Close();
}//end using resp
pipe.Send("Api Call complete; Parsing returned data...");
int i = 0;
String h = "";
String l = "";
int s = 0;
int p = 0;
int b = 0;
int payload = 0;
foreach (string line in json.Split(new[] { "}," }, StringSplitOptions.None))
{
if (line != "")
{
l = line;
i = l.Replace("{", "").Length + 1;
p = l.LastIndexOf("{");
if (line.Length > i) //we find this at the beginning of a group of arrays
{
h = line.Substring(0, p - 1);
s = Math.Max(0, h.LastIndexOf("{"));
if (h.Length > s && s != 0)
/*We have a nested array that has more than one level.
*This should only occur at the beginning of new array group.
*Advance the payload counter and get the correct string from line.*/
{
payload++;
l = line.Substring(s, line.Length - s);
}
h = (s >= 0) ? h.Substring(0, s) : h;
//=============
/*At this point 'h' is a nest collection. Split and add to table.*/
string[] OrderedNest = h.Split('{');
for (int z = 0; z < OrderedNest.Length; z++)
{
if (OrderedNest[z] != "")
{
table.Rows.Add(payload, "{" + OrderedNest[z].Replace(":", "").Replace("[","").Replace("]","") + "}");
}
}
//=============
}
else
{
h = null;
}
//at this point the first character in the row should be a "{"; If not we need to add one.
if (l[0].ToString() != "{")
{
l = "{" + l;
}
if (l.Replace("{", "").Length != l.Replace("}", "").Length) //opening and closing braces don't match; match the closing to the opening
{
l = l.Replace("}", "");
b = l.Length - l.Replace("{", "").Length;
l = l + new String('}', b);
}
table.Rows.Add(payload, l.Replace("\\\"", "").Replace("\\", "").Replace("]","").Replace("[",""));
}
}
//====
using (SqlConnection cnn = new SqlConnection(connectionString))
{
cnn.Open();
using (SqlBulkCopy copy = new SqlBulkCopy(cnn))
{
copy.DestinationTableName = "sp_WebSvcs.dbo.JSON_DATA";
copy.WriteToServer(table);
}
}
//====
} //end try
catch (Exception e)
{
pipe.Send("We have a problem!");
throw new Exception("\n\n" + e.Message + "\n\n");
}
pipe.Send("Parsing complete");
}