C# 以编程方式创建没有源或目标的数据流任务
我有一个SSIS包,它将数据从Microsoft Access复制到SQL Server。这两套表格几乎完全相同 背景:毫不奇怪,随着产品的开发,我们的表模式也在增长。因此,我们一直需要使用新列更新SSIS包。这是一项非常枯燥的任务,因此我尝试用C#以编程方式创建SSIS包。这进展顺利,但我想让这个过程更容易 问题:包生成过程要求源(访问)和目标(SQL Server)在运行C#时存在。这与我们当前的流程不太匹配。因此,我想:C# 以编程方式创建没有源或目标的数据流任务,c#,ssis,C#,Ssis,我有一个SSIS包,它将数据从Microsoft Access复制到SQL Server。这两套表格几乎完全相同 背景:毫不奇怪,随着产品的开发,我们的表模式也在增长。因此,我们一直需要使用新列更新SSIS包。这是一项非常枯燥的任务,因此我尝试用C#以编程方式创建SSIS包。这进展顺利,但我想让这个过程更容易 问题:包生成过程要求源(访问)和目标(SQL Server)在运行C#时存在。这与我们当前的流程不太匹配。因此,我想: 以编程方式提供表和列(SQL和Access表是从XML元数据生成的
- 以编程方式提供表和列(SQL和Access表是从XML元数据生成的,因此我们可以从相同的元数据向SSI提供表和列)
- 加载包时,运行一些C#来修改包,即重新初始化元数据等
public IDTSComponentMetaData100 AddDestAdapter(IDTSPipeline100 pipeline, ConnectionManager destConnMgr, out IDTSDesigntimeComponent100 destDesignTimeComp)
{
IDTSComponentMetaData100 destComp = pipeline.ComponentMetaDataCollection.New();
destComp.ComponentClassID = OLEDB_DEST_GUID;
destComp.ValidateExternalMetadata = true;
destDesignTimeComp = destComp.Instantiate();
destDesignTimeComp.ProvideComponentProperties();
destComp.Name = "OleDB Destination - Sql Server";
destDesignTimeComp.SetComponentProperty("AccessMode", 0);
destDesignTimeComp.SetComponentProperty("OpenRowset", quotedTableName);
// set connection
destComp.RuntimeConnectionCollection[0].ConnectionManager = DtsConvert.GetExtendedInterface(destConnMgr);
destComp.RuntimeConnectionCollection[0].ConnectionManagerID = destConnMgr.ID;
// get metadata
destDesignTimeComp.AcquireConnections(null);
destDesignTimeComp.ReinitializeMetaData();
destDesignTimeComp.ReleaseConnections();
extCols = destComp.InputCollection[0].ExternalMetadataColumnCollection;
return destComp;
}
public void AddPathsAndConnectColumns()
{
IDTSOutput100 srcOutput = srcComp.OutputCollection[0];
IDTSOutputColumnCollection100 srcOutputCols = srcOutput.OutputColumnCollection;
IDTSInput100 destInput = destComp.InputCollection[0];
IDTSInputColumnCollection100 destInputCols = destInput.InputColumnCollection;
IDTSExternalMetadataColumnCollection100 destExtCols = destInput.ExternalMetadataColumnCollection;
Hashtable destColtable = new Hashtable(destExtCols.Count);
foreach (IDTSExternalMetadataColumn100 extCol in destExtCols)
{
destColtable.Add(extCol.Name, extCol);
}
// colConvertTable stores a pair of columns which need a type conversion
// colConnectTable stores a pair of columns which dont need a type conversion and can be connected directly.
Hashtable colConvertTable = new Hashtable(srcOutputCols.Count);
Hashtable colConnectTable = new Hashtable(srcOutputCols.Count);
foreach (IDTSOutputColumn100 outputCol in srcOutputCols)
{
// Get the column name to look for in the destination.
// Match column by name if match table is not used.
String colNameToLookfor = outputCol.Name;
IDTSExternalMetadataColumn100 extCol = (String.IsNullOrEmpty(colNameToLookfor)) ? null : (IDTSExternalMetadataColumn100)destColtable[colNameToLookfor];
// Does the destination column exist?
if (extCol != null)
{
colConnectTable.Add(outputCol.ID, extCol);
}
}
// Convert transform not needed. Connect src and destination directly.
pipeline.PathCollection.New().AttachPathAndPropagateNotifications(srcOutput, destInput);
IDTSVirtualInput100 destVirInput = destInput.GetVirtualInput();
foreach (object key in colConnectTable.Keys)
{
int colID = (int)key;
IDTSExternalMetadataColumn100 extCol = (IDTSExternalMetadataColumn100)colConnectTable[key];
// Create an input column from an output col of previous component.
destVirInput.SetUsageType(colID, DTSUsageType.UT_READONLY);
IDTSInputColumn100 inputCol = destInputCols.GetInputColumnByLineageID(colID);
if (inputCol != null)
{
// map the input column with an external metadata column
destDesignTimeComp.MapInputColumn(destInput.ID, inputCol.ID, extCol.ID);
}
}
}
可能的勘探路线:
- 是否有人有同样的问题(尽管没有具体的解决方案)
- 从对BIML的一些调查来看,似乎(离线模式似乎可行,尽管我有点担心,因为我在互联网上找不到一个例子)
namespace ConsoleApplication5
{
// A struct ot represent an external column
public struct Column
{
public String Name;
public String SSISDataType;
public int Length;
public int Precision;
public int Scale;
public int CodePage;
public Column(String name, String ssisDataType, int length, int precision, int scale, int codePage)
{
Name = name;
SSISDataType = ssisDataType;
Length = length;
Precision = precision;
Scale = scale;
CodePage = codePage;
}
}
public class Packager
{
public Packager()
{
build();
}
private void build()
{
#region Package Related
// Package related
Package package = new Package();
Executable e = package.Executables.Add("STOCK:PipelineTask");
TaskHost thMainPipe = e as TaskHost;
MainPipe dataFlowTask = thMainPipe.InnerObject as MainPipe;
thMainPipe.Name = "MyDFT";
thMainPipe.DelayValidation = true;
#endregion
#region Add Connection Manager
// Add Connection Manager
ConnectionManager cm = package.Connections.Add("OLEDB");
cm.Name = "OLEDB ConnectionManager";
cm.ConnectionString = "Data Source=(local);" +
"Initial Catalog=AdventureWorks;Provider=SQLOLEDB.1;" +
"Integrated Security=SSPI;";
#endregion
#region Add a OleDB Source and set up basic properties
// Add an OLE DB source to the data flow.
IDTSComponentMetaData100 component = dataFlowTask.ComponentMetaDataCollection.New();
component.Name = "OLEDBSource";
component.ComponentClassID = "Microsoft.OLEDBSource"; // check for the exact component class ID on your machine
// Get the design time instance of the component.
CManagedComponentWrapper instance = component.Instantiate();
// Initialize the component
instance.ProvideComponentProperties();
// Specify the connection manager.
if (component.RuntimeConnectionCollection.Count > 0)
{
component.RuntimeConnectionCollection[0].ConnectionManager = DtsConvert.GetExtendedInterface(package.Connections[0]);
component.RuntimeConnectionCollection[0].ConnectionManagerID = package.Connections[0].ID;
}
// Set the custom properties.
instance.SetComponentProperty("AccessMode", 2);
instance.SetComponentProperty("SqlCommand", "Select * from Production.Product");
#endregion
#region Core example showcasing use of IDTSExternalMetadataColumn when external data source is not available.
// Typically here we call acquireconnection, reinitmetadata etc to get the metadata from a data source that exists.
// Instead we will populate the metadata ourselves
#region Get External Columns Metadata
// Get the collection of external columns
List<Column> externalColumns = new List<Column>();
// Hard Coding Here. But grab them from your metadata source programmatically.
Column columnA = new Column("col_a", "DT_STR", 24, 0, 0, 1252);
Column columnB = new Column("col_b", "DT_STR", 36, 0, 0, 1252);
Column columnC = new Column("col_c", "DT_STR", 48, 0, 0, 1252);
externalColumns.Add(columnA);
externalColumns.Add(columnB);
externalColumns.Add(columnC);
#endregion
#region Add External Columns to our required IDTSOutput100
// Grab the appropriate output as needed. We will be adding ExternalColumns to this Output
IDTSOutput100 output = component.OutputCollection[0];
// Add each external column to the above IDTSOutPut
foreach (Column extCol in externalColumns)
{
IDTSExternalMetadataColumn100 col = output.ExternalMetadataColumnCollection.New();
col.Name = extCol.Name;
col.Scale = extCol.Scale;
col.Precision = extCol.Precision;
col.Length = extCol.Length;
col.CodePage = extCol.CodePage;
col.DataType = (Wrapper.DataType)Enum.Parse(typeof(Wrapper.DataType), extCol.SSISDataType);
}
#endregion
#region Create OutputColumn if it does not exist/or grab the output column if it Exists. Then associate it to the External Column
// Now associate the External Column to an Output Column.
// Here, we will simply associate the external column to an output column if the name matches (because of our use case)
foreach (IDTSExternalMetadataColumn100 extCol in output.ExternalMetadataColumnCollection)
{
bool outputColExists = false;
// Set DataTypes and Associate with external column if output col exists
foreach (IDTSOutputColumn100 outputCol in output.OutputColumnCollection)
{
if (outputCol.Name == extCol.Name) // is map based on name
{
// Set the data type properties
outputCol.SetDataTypeProperties(extCol.DataType, extCol.Length, extCol.Precision, extCol.Scale, extCol.CodePage);
// Associate the external column and the output column
outputCol.ExternalMetadataColumnID = extCol.ID;
outputColExists = true;
break;
}
}
// Create an IDTSOutputColumn if not exists.
if (!(outputColExists))
{
IDTSOutputColumn100 outputCol = output.OutputColumnCollection.New();
outputCol.Name = extCol.Name; // map is based on name
// Set the data type properties
outputCol.SetDataTypeProperties(extCol.DataType, extCol.Length, extCol.Precision, extCol.Scale, extCol.CodePage);
// Associate the external column and the output column
outputCol.ExternalMetadataColumnID = extCol.ID;
}
}
#endregion
#endregion
#region Save the Package to disk
new Application().SaveToXml(@"C:\Temp\Pkg.dtsx", package, null);
#endregion
}
}
命名空间控制台应用程序5
{
//表示外部列的结构
公共结构列
{
公共字符串名称;
公共字符串数据类型;
公共整数长度;
公共整数精度;
公共int量表;
公共int码页;
公共列(字符串名称、字符串数据类型、整数长度、整数精度、整数比例、整数代码页)
{
名称=名称;
SSISDataType=SSISDataType;
长度=长度;
精度=精度;
比例=比例;
代码页=代码页;
}
}
公营包装商
{
公共包装商()
{
build();
}
私有void build()
{
#区域包相关
//包装相关
包=新包();
可执行文件e=package.Executables.Add(“STOCK:PipelineTask”);
TaskHost thMainPipe=e作为TaskHost;
MainPipe dataFlowTask=thMainPipe.InnerObject作为MainPipe;
thMainPipe.Name=“MyDFT”;
thMainPipe.DelayValidation=true;
#端区
#区域添加连接管理器
//添加连接管理器
ConnectionManager cm=package.Connections.Add(“OLEDB”);
cm.Name=“OLEDB连接管理器”;
cm.ConnectionString=“数据源=(本地);”+
“初始目录=AdventureWorks;提供程序=SQLOLEDB.1;”+
“综合安全=SSPI;”;
#端区
#区域添加OleDB源并设置基本属性
//将OLE DB源添加到数据流中。
IDTSComponentMetaData100组件=dataFlowTask.ComponentMetaDataCollection.New();
component.Name=“OLEDBSource”;
component.ComponentClassID=“Microsoft.OLEDBSource”;//检查计算机上的确切组件类ID
//获取组件的设计时实例。
CManagedComponentWrapper实例=组件。实例化();
//初始化组件
providDecomponentProperties();
//指定连接管理器。
如果(component.RuntimeConnectionCollection.Count>0)
{
component.RuntimeConnectionCollection[0].ConnectionManager=DtsConvert.GetExtendedInterface(package.Connections[0]);
component.RuntimeConnectionCollection[0]。ConnectionManagerID=package.Connections[0]。ID;
}
//设置自定义属性。
SetComponentProperty(“AccessMode