SSIS Team Blog : Samples

API Sample – Lookup Transform

mmasson — Fri, 02 Jan 2009 21:34:50 GMT

This sample creates a data flow package with an OLEDB Source component feeding into a Lookup Transform. The Lookup transform is set to Full Cache mode, and uses [DimCustomer] as its reference table.

Items of interest:

CustomerKey and GeographyKey are used as the index (join) columns. This is configured by using the JoinToReferenceColumn property
The FirstName column is being overwritten by the value retrieved by the lookup transform
The LastName2 column is being added as a new output column

static void Main(string[] args)
{
    Package package = new Package();

    // Add Data Flow Task
    Executable dataFlowTask = package.Executables.Add("STOCK:PipelineTask");

    // Set the name (otherwise it will be a random GUID value)
    TaskHost taskHost = dataFlowTask as TaskHost;
    taskHost.Name = "Data Flow Task";

    // We need a reference to the InnerObject to add items to the data flow
    MainPipe pipeline = taskHost.InnerObject as MainPipe;

    //
    // Add connection manager
    //

    ConnectionManager connection = package.Connections.Add("OLEDB");
    connection.Name = "localhost";
    connection.ConnectionString = "Data Source=localhost;Initial Catalog=AdventureWorksDW2008;Provider=SQLNCLI10.1;Integrated Security=SSPI;Auto Translate=False;";

    //
    // Add OLEDB Source
    //

    IDTSComponentMetaData100 srcComponent = pipeline.ComponentMetaDataCollection.New();
    srcComponent.ComponentClassID = "DTSAdapter.OleDbSource";
    srcComponent.ValidateExternalMetadata = true;
    IDTSDesigntimeComponent100 srcDesignTimeComponent = srcComponent.Instantiate();
    srcDesignTimeComponent.ProvideComponentProperties();
    srcComponent.Name = "OleDb Source";

    // Configure it to read from the given table
    srcDesignTimeComponent.SetComponentProperty("AccessMode", 0);
    srcDesignTimeComponent.SetComponentProperty("OpenRowset", "[DimCustomer]");

    // Set the connection manager
    srcComponent.RuntimeConnectionCollection[0].ConnectionManager = DtsConvert.GetExtendedInterface(connection);
    srcComponent.RuntimeConnectionCollection[0].ConnectionManagerID = connection.ID;

    // Retrieve the column metadata
    srcDesignTimeComponent.AcquireConnections(null);
    srcDesignTimeComponent.ReinitializeMetaData();
    srcDesignTimeComponent.ReleaseConnections();

    // Add transform
    IDTSComponentMetaData100 lookupComponent = pipeline.ComponentMetaDataCollection.New();
    lookupComponent.ComponentClassID = "DTSTransform.Lookup";
    lookupComponent.Name = "Lookup";

    CManagedComponentWrapper lookupWrapper = lookupComponent.Instantiate();
    lookupWrapper.ProvideComponentProperties();

    // Connect the source and the transform
    IDTSPath100 path = pipeline.PathCollection.New();
    path.AttachPathAndPropagateNotifications(srcComponent.OutputCollection[0], lookupComponent.InputCollection[0]);

    //
    // Configure the transform
    //

    // Set the connection manager
    lookupComponent.RuntimeConnectionCollection[0].ConnectionManager = DtsConvert.GetExtendedInterface(connection);
    lookupComponent.RuntimeConnectionCollection[0].ConnectionManagerID = connection.ID;

    // Cache Type - Full = 0, Partial = 1, None = 2
    lookupWrapper.SetComponentProperty("CacheType", 0);
    lookupWrapper.SetComponentProperty("SqlCommand", "select * from [DimCustomer]");

    // initialize metadata
    lookupWrapper.AcquireConnections(null);
    lookupWrapper.ReinitializeMetaData();
    lookupWrapper.ReleaseConnections();

    // Mark the columns we are joining on
    IDTSInput100 lookupInput = lookupComponent.InputCollection[0];
    IDTSInputColumnCollection100 lookupInputColumns = lookupInput.InputColumnCollection;
    IDTSVirtualInput100 lookupVirtualInput = lookupInput.GetVirtualInput();
    IDTSVirtualInputColumnCollection100 lookupVirtualInputColumns = lookupVirtualInput.VirtualInputColumnCollection;

    // We are joining on CustomerKey and GeographyKey
    // Note: join columns should be marked as READONLY
    var joinColumns = new string[] { "CustomerKey", "GeographyKey" };
    foreach (string columnName in joinColumns)
    {
        IDTSVirtualInputColumn100 virtualColumn = lookupVirtualInputColumns[columnName];
        IDTSInputColumn100 inputColumn = lookupWrapper.SetUsageType(lookupInput.ID, lookupVirtualInput, virtualColumn.LineageID, DTSUsageType.UT_READONLY);
        lookupWrapper.SetInputColumnProperty(lookupInput.ID, inputColumn.ID, "JoinToReferenceColumn", columnName);
    }
    
    // Overwrite the existing FirstName column value with the one returned by the Lookup.
    // To do this, we need to flag the column as READWRITE, and set the CopyFromReferenceColumn property on the input
    var overwriteColumns = new string[] { "FirstName" };
    foreach (string columnName in overwriteColumns)
    {
        IDTSVirtualInputColumn100 virtualColumn = lookupVirtualInputColumns[columnName];
        IDTSInputColumn100 inputColumn = lookupWrapper.SetUsageType(lookupInput.ID, lookupVirtualInput, virtualColumn.LineageID, DTSUsageType.UT_READWRITE);

        lookupWrapper.SetInputColumnProperty(lookupInput.ID, inputColumn.ID, "CopyFromReferenceColumn", columnName);
    }

    // First output is the Match output
    IDTSOutput100 lookupMatchOutput = lookupComponent.OutputCollection[0];

    // Add a new LastName2 column from the "LastName" column returned by the lookup
    var newColumns = new Dictionary<string, string>();
    newColumns.Add("LastName", "LastName2");

    foreach (string sourceColumn in newColumns.Keys)
    {
        string newColumnName = newColumns[sourceColumn];
        string description = string.Format("Copy of {0}", sourceColumn);

        // insert the new column
        IDTSOutputColumn100 outputColumn = lookupWrapper.InsertOutputColumnAt(lookupMatchOutput.ID, 0, newColumnName, description);
        lookupWrapper.SetOutputColumnProperty(lookupMatchOutput.ID, outputColumn.ID, "CopyFromReferenceColumn", sourceColumn);
    }
}

API Sample – Row Count Transform

mmasson — Wed, 31 Dec 2008 01:47:13 GMT

This sample creates a data flow package with an OLEDB Source that feeds into a Row Count transform. The Row Count transform is configured to store the result in the RowCountVar package variable.

static void Main(string[] args)
{
    Package package = new Package();

    // Add Data Flow Task
    Executable dataFlowTask = package.Executables.Add("STOCK:PipelineTask");

    // Set the name (otherwise it will be a random GUID value)
    TaskHost taskHost = dataFlowTask as TaskHost;
    taskHost.Name = "Data Flow Task";

    // We need a reference to the InnerObject to add items to the data flow
    MainPipe pipeline = taskHost.InnerObject as MainPipe;

    //
    // Create a package variable to store the row count value
    //

    package.Variables.Add("RowCountVar", false, "User", 0);

    //
    // Add connection manager
    //

    ConnectionManager connection = package.Connections.Add("OLEDB");
    connection.Name = "localhost";
    connection.ConnectionString = "Data Source=localhost;Initial Catalog=AdventureWorksDW2008;Provider=SQLNCLI10.1;Integrated Security=SSPI;Auto Translate=False;";

    //
    // Add OLEDB Source
    //

    IDTSComponentMetaData100 srcComponent = pipeline.ComponentMetaDataCollection.New();
    srcComponent.ComponentClassID = "DTSAdapter.OleDbSource";
    srcComponent.ValidateExternalMetadata = true;
    IDTSDesigntimeComponent100 srcDesignTimeComponent = srcComponent.Instantiate();
    srcDesignTimeComponent.ProvideComponentProperties();
    srcComponent.Name = "OleDb Source";

    // Configure it to read from the given table
    srcDesignTimeComponent.SetComponentProperty("AccessMode", 0);
    srcDesignTimeComponent.SetComponentProperty("OpenRowset", "[DimCustomer]");

    // Set the connection manager
    srcComponent.RuntimeConnectionCollection[0].ConnectionManager = DtsConvert.GetExtendedInterface(connection);
    srcComponent.RuntimeConnectionCollection[0].ConnectionManagerID = connection.ID;

    // Retrieve the column metadata
    srcDesignTimeComponent.AcquireConnections(null);
    srcDesignTimeComponent.ReinitializeMetaData();
    srcDesignTimeComponent.ReleaseConnections();

    //
    // Add Row Count transform
    //

    IDTSComponentMetaData100 rowCount = pipeline.ComponentMetaDataCollection.New();
    rowCount.ComponentClassID = "DTSTransform.RowCount";
    CManagedComponentWrapper instance = rowCount.Instantiate();
    instance.ProvideComponentProperties();

    // Set the variable name property
    instance.SetComponentProperty("VariableName", "User::RowCountVar");

    //
    // Connect the OLEDB Source and the Row Count
    //

    IDTSPath100 path = pipeline.PathCollection.New();
    path.AttachPathAndPropagateNotifications(srcComponent.OutputCollection[0], rowCount.InputCollection[0]);
}

API Sample – ADO.Net Source

mmasson — Wed, 31 Dec 2008 01:38:26 GMT

This sample creates a data flow package with an ADO.Net source.

Items of interest:

The AccessMode property is similar to that of the OleDB Source, except it only has two values – 0 for TableOrViewName, and 2 for SqlCommand. Examples of both are included in the sample (one is commented out).
To be able to reference the DataReaderSourceAdapter (the class name of the ADO.Net Source), you’ll need to add a reference to the ADONETSrc assembly - C:\Program Files\Microsoft SQL Server\100\DTS\PipelineComponents\Microsoft.SqlServer.ADONETSrc.dll

static void Main(string[] args)
{
    Package package = new Package();

    // Add Data Flow Task
    Executable dataFlowTask = package.Executables.Add("STOCK:PipelineTask");

    // Set the name (otherwise it will be a random GUID value)
    TaskHost taskHost = dataFlowTask as TaskHost;
    taskHost.Name = "Data Flow Task";

    // We need a reference to the InnerObject to add items to the data flow
    MainPipe pipeline = taskHost.InnerObject as MainPipe;

    // Create ADO.Net connection manager
    string connectionType = string.Format("ADO.NET:{0}", typeof(SqlConnection).AssemblyQualifiedName);
    ConnectionManager connection = package.Connections.Add(connectionType);
    connection.Name = "localhost";
    connection.ConnectionString = "Data Source=localhost;Initial Catalog=AdventureWorksDW2008;Integrated Security=True;";

    // Add the ADO.NET Source
    IDTSComponentMetaData100 src = pipeline.ComponentMetaDataCollection.New();
    src.Name = "AdoNet Source";
    src.ComponentClassID = typeof(DataReaderSourceAdapter).AssemblyQualifiedName;
    src.ValidateExternalMetadata = true;
    IDTSDesigntimeComponent100 instance = src.Instantiate();
    instance.ProvideComponentProperties();

    // Configure the source
    //instance.SetComponentProperty("AccessMode", 2);
    //instance.SetComponentProperty("SqlCommand", "select * from [DimCustomer]");
    instance.SetComponentProperty("AccessMode", 0);
    instance.SetComponentProperty("TableOrViewName", "\"dbo\".\"DimCustomer\"");

    // Set the connection manager
    src.RuntimeConnectionCollection[0].ConnectionManager = DtsConvert.GetExtendedInterface(connection);
    src.RuntimeConnectionCollection[0].ConnectionManagerID = connection.ID;

    // Retrieve the column metadata
    instance.AcquireConnections(null);
    instance.ReinitializeMetaData();
    instance.ReleaseConnections();
}

EzAPI – Alternative package creation API

mmasson — Wed, 31 Dec 2008 00:20:16 GMT

SSIS provides APIs to programmatically create dtsx packages. While we provide a managed wrapper layer for the runtime/control flow, you have to use the lower level COM wrappers (DTSPipelineWrap) to create your data flows – the usability of which could definitely be improved.

Evgeny Koblov, one of the testers on the SSIS team, has put together a really powerful framework for generating packages called EzAPI. It’s been used by the test team internally for awhile now, and we’ve now published it to Codeplex as well. The project includes a pretty detailed readme file, as well as some great samples on how to use the classes.

This post is mostly a formatted version of the readme file written by Evgeny. It is available in the root directory of the project’s source code.

Overview

This sample provides some functionality to easily create SSIS packages programmatically and dynamically alter their objects (tasks, components, changing metadata, connection strings, etc). This framework supports:

Creation SSIS packages of any complexity including both SSIS runtime and pipeline (tasks, containers and components)
BIDS like behavior (automatic column mapping in destinations, automatic metadata refresh, default values of properties, etc)

Requirements

To use EzAPI framework and be able to compile and run the demo applications, the following components are required:

The sample must be installed on the local hard drive
You must install the common tools for SQL Server 2008
You must install SQL Server Integration Services 2008
Adventure Works sample database installed under a default instance on the local machine

Installer

The installer will place source and project files into the chosen directory. Additionally, pre-built binaries are placed into the Global Assembly Cache (GAC) and into the C:\Program Files\Microsoft SQL Server\100\DTS\Binn folder

Background

Having an easy way to create SSIS packages programmatically is vital. In some cases static packages do not provide enough flexibility – it is hard to change property task or component of a static package using SSIS object model while using configurations is not always possible. Current approach to create SSIS package programmatically is to use SSIS object model directly. However SSIS object SSIS object model was designed to be universal for all kinds of components (native and managed ones) and to separate runtime and design time phases of component usage. However both of the stages use the same metadata. This is the reason why SSIS distinguishes design-time, runtime functionality and metadata. The side effect is that more code is necessary to create a package and once the package is created – it is not very convenient to change properties of pipeline components and especially metadata of input, output, virtual and external columns.

However, in .NET managed world we can use all the advantages of OOP and have a corresponding class that will encapsulate both metadata and operations that can be made on them – so, there is no need to separate it as we do not deal with runtime internals during design phase.

The framework proposed in this sample can significantly reduce the amount of code that need to be supported and allows more efficient development of SSIS packages programmatically.

EzAPI Architecture

EzAPI takes SSIS COM object (implemented via many C++ and C# classes) as a single entity. It means it maps each EzAPI object to each SSIS COM object. Here is how it approximately looks like for control flow and for data flow:

Fig 1a – class hierarchy for SSIS runtime

Fig 1b – class hierarchy for SSIS pipeline

In the control flow – the base entity is executable, that can be either a container or a task. So EzContainer and EzTask classes derive from EzExecutable class. Then all the containers derive from EzContainer and all tasks derive from EzTask.

In the pipeline – component is the base entity. We have three types of components (can be easily noticed in BIDS): Sources, Destinations and Transformations. Sources and Destinations are actually adapters that connect SSIS with some input or output (Database, Flat File, Variable, etc) That’s why they share some functionality and derive from EzAdapter class.

This approach allows accumulating common functionality in the base classes on each level of hierarchy. When you use SSIS object model or PackageBuilder you should constantly duplicate code that retrieves instances of SSIS objects, their metadata, implements column mapping logic (EzAPI allows both – default , BIDS-like mapping, and custom mapping via special functions MapColumn and UnmapColumn), etc.

Using EzAPI

This section contains some examples of how EzAPI can be used

To design a package using EzAPI:

Create a new .NET project.
Add reference to EzAPI.dll.
Add using entry: using Microsoft.SqlServer.SSIS.EzAPI;

Creation of package with single task

The example below creates packages with ExecutePackageTask:

using Microsoft.SqlServer.SSIS.EzAPI;
…

namespace ConsoleApplication1
{
    // DEMO1: This package contains a single Execute package task
    public class EzExecPkgPackage : EzPackage
    {
        // Provide this constructor only if you want to overload Assignment operator
        public EzExecPkgPackage(Package p) : base(p) { }   // this constructor MUST BE present

        // Provide assignment operator if you want to be able to Assign SSIS Package to EzPackage
        public static implicit operator EzExecPkgPackage(Package p) { return new EzExecPkgPackage(p); }

        // All the tasks, components and connection managers which should be linked to the corresponding
        // SSIS package objects MUST BE PUBLIC MEMBERS if you want to be able to assign SSIS package to EzPackage
        public EzExecPackage ExecPkg;
        public EzFileCM PkgCM;

        public EzExecPkgPackage(string pkgName)
            : base()
        {
            PkgCM = new EzFileCM(this);
            PkgCM.ConnectionString = pkgName;
            PkgCM.Name = "PackageConnection";
            ExecPkg = new EzExecPackage(this);
            ExecPkg.Name = "ExecutePackage";
            ExecPkg.Connection = PkgCM;
        }

        [STAThread]
        static void Main(string[] args)
        {
            // DEMO 1
            EzPackage p = new EzPackage();
            p.SaveToFile("testpkg.dtsx");
            EzExecPkgPackage p1 = new EzExecPkgPackage("testpkg.dtsx");
            p1.SaveToFile("demo1.dtsx");
            p1.Execute();
            Console.Write(string.Format("Package1 executed with result {0}\n", p1.ExecutionResult));
        }
    }
}

This code will create package with a single ExecutePackageTask:

Creation of package with simple dataflow

The next example shows how to create a package with a simple dataflow that has OleDB source and FlatFile destination.

namespace ConsoleApplication1
{
    // DEMO2: Simple data transfer from source to destination. Use EzSrcDestPackage template for this
    public class EzOleDbToFilePackage : EzSrcDestPackage<EzOleDbSource, EzSqlOleDbCM, EzFlatFileDestination, EzFlatFileCM>
    {
        public EzOleDbToFilePackage(Package p) : base(p) { }
        public static implicit operator EzOleDbToFilePackage(Package p) { return new EzOleDbToFilePackage(p); }

        public EzOleDbToFilePackage(string srv, string db, string table, string file)
            : base()
        {
            SrcConn.SetConnectionString(srv, db);
            Source.Table = table;
            DestConn.ConnectionString = file;
            Dest.Overwrite = true;
            // This method defines the columns in FlatFile connection manager which have the same
            // datatypes as flat file destination
            Dest.DefineColumnsInCM();
        }

        [STAThread]
        static void Main(string[] args)
        {
            // DEMO 2
            EzOleDbToFilePackage p2 = new EzOleDbToFilePackage("localhost", "AdventureWorks", "Address", "result.txt");
            p2.DataFlow.Disable = true;
            p2.Execute();
            Console.Write(string.Format("Package2 executed with result {0}\n", p2.ExecutionResult));
        }
    }
}

Dataflow task of the created package looks like this:

Notice that the components are already attached, fields in connection managers are defined. Also Input and external columns with the same names are automatically mapped in destination:

Another thing that is worth mentioning is that in this example a template class was used:

EzSrcDestPackage<EzOleDbSource, EzSqlOleDbCM, EzFlatFileDestination, EzFlatFileCM>

Here is where EzAPI becomes really powerful. Even this simple template class allows creation of packages with dataflow task with any source and any destination, by just specifying template parameters. As for all the properties of package, dataflow task, or components – they can be set directly in the testcase, instead of passing them to “builder” using helper classes.

Package with DataFlow transformation

Now let’s create a package that has structure used by most test suites that create packages programmatically, i.e. package with dataflow: source->transform->destination:

class EzMyPackage<EzTransform> : EzTransformPackage<EzOleDbSource, EzSqlOleDbCM, EzTransform, EzFlatFileDestination, EzFlatFileCM> where EzTransform: EzComponent
{
    // These two methods are not deployed as I am not going to assign SSIS package to EzSortPackage in demo
    public EzMyPackage(Package p) : base(p) { }
    public static implicit operator EzMyPackage<EzTransform>(Package p) { return new EzMyPackage<EzTransform>(p); }

    public EzMyPackage(string srv, string db, string sql, string file) : base()
    {
        SrcConn.SetConnectionString(srv, db);
        Source.SqlCommand = sql;
        DestConn.ConnectionString = file;
        Dest.Overwrite = true;
        Dest.DefineColumnsInCM();
    }

}

And now – this is how package with Sort Transform can be created and used:

public static void Main(string[] args)
{
    // DEMO 3
    EzMyPackage<EzSortTransform> p3 = new EzMyPackage<EzSortTransform>("localhost", "AdventureWorks", "select * from Person.Address", "result1.txt");
    p3.Transform.EliminateDuplicates = true;
    p3.Transform.SortOrder["AddressID"] = -1; // sort in descending order
    p3.SaveToFile("demo3.dtsx");
    p3.Execute();
    Console.Write(string.Format("Package3 executed with result {0}\n", p3.ExecutionResult));
    // Assign SSIS package to EzPackage
    p3 = new Application().LoadPackage("demo3.dtsx", null);
    p3.Execute();
    Console.Write(string.Format("Package3 executed with result {0}\n", p3.ExecutionResult));
}

And this is how to turn it into package with DerivedColumn transform:

public static void Main(string[] args)
{
    EzMyPackage<EzDerivedColumn> p = new EzMyPackage<EzDerivedColumn>("localhost", "AdventureWorks", "Address", "c:\\result1.txt");
    p.Transform.Expression["SmallDate1"] = "DATEADD(\"Year\",1,SmallDate1)"; // Setting derived column expressions
    p.Transform.Expression["Date1"] = "DATEADD(\"Year\",1,Date1)";
    p.Transform.Expression["SmallDate2"] = "DATEADD(\"Quarter\",2,SmallDate2)";
    p.Transform.Expression["Date2"] = "DATEADD(\"Quarter\",2,Date2)";
    p.Transform.Expression["SmallDate3"] = "DATEADD(\"Month\",4,SmallDate3)";
    p.Transform.Expression["Date3"] = "DATEADD(\"Month\",4,Date3)";
    p.SaveToFile(@"C:\temp\ezdemo\demo3.dtsx");
    p.Execute();
    Console.Write(string.Format("Package3 executed with result {0}\n", p.ExecutionResult));
    p = new Application().LoadPackage(@"C:\temp\ezdemo\demo3.dtsx", null); //assign SSIS package to EzPackage
    p.Execute();
    Console.Write(string.Format("Package3 executed with result {0}\n", p.ExecutionResult));
}

We didn’t have to create a separate class for that – we could use existing template without introducing any modifications.

Package with Loop containers

Another possible scenario is execution of some process in a loop. EzAPI provides this possibility and here is the example:

public class EzLoopSortPackage : EzLoopTransformPackage<EzOleDbSource, EzSqlOleDbCM, EzSortTransform, EzFlatFileDestination, EzFlatFileCM>
{
    public EzLoopSortPackage(Package p) : base(p) { }
    public static implicit operator EzLoopSortPackage(Package p) { return new EzLoopSortPackage(p); }

    public EzLoopSortPackage(string srv, string db, string table, string file)
        : base()
    {
        SrcConn.SetConnectionString(srv, db);
        Source.Table = table;
        Source.AccessMode = AccessMode.AM_OPENROWSET;
        DestConn.ConnectionString = file;
        Dest.Overwrite = true;
        Dest.DefineColumnsInCM();
    }

    [STAThread]
    static void Main(string[] args)
    {
        EzLoopSortPackage p5 = new EzLoopSortPackage("localhost", "AdventureWorks", "Address", "result1.txt");
        p5.Transform.MaxThreads = -1; // Do not limit number of threads
        p5.Transform.EliminateDuplicates = true;
        p5.Transform.SortOrder["AddressID"] = -1; // sort in descending order
        p5.Variables.Add("LoopCounter", false, "User", 0);
        p5.ForLoop.InitExpression = "@[User::LoopCounter] = 0";
        p5.ForLoop.AssignExpression = "@[User::LoopCounter] = @[User::LoopCounter] + 1";
        p5.ForLoop.EvalExpression = "@[User::LoopCounter] < 3";
        p5.Execute();
        Console.Write(string.Format("Package5 executed with result {0}\n", p5.ExecutionResult));
    }
}

This example creates a package with dataflow task executed in a for loop 3 times. The code above is pretty straightforward, all the property names are the same as they are in BIDS. Here is the package created:

Package with Multiple dataflows

Another important scenario is package with multiple dataflows. A typical example here is lookup transform with full or partial cache.

namespace ConsoleApplication1
{
    public class EzLookupCachePkg : EzPkgWithExec<EzTransformDF<EzOleDbSource, EzSqlOleDbCM, EzCacheTransform>>
    {
        public EzTransDestConnDF<EzOleDbSource, EzSqlOleDbCM, EzLookup, EzFlatFileDestination, EzFlatFileCM> LookupDF;
        public EzLookupCachePkg(string srv, string srcDb, string refDb, string refSql) : base()
        {
            LookupDF = new EzTransDestConnDF<EzOleDbSource, EzSqlOleDbCM, EzLookup, EzFlatFileDestination, EzFlatFileCM>(this);
            LookupDF.AttachTo(Exec);
            Exec.Transform.Connection = new EzCacheCM(this);
            LookupDF.Transform.CacheConnection = Exec.Transform.Connection;
            Exec.Transform.Connection.Name = "Cache";
            Variables.Add("CheckSum0", false, "", new byte[] { });
            Exec.Name = "CacheDF";
            Exec.SrcConn.Name = "RefDb";
            Exec.SrcConn.SetConnectionString(srv, refDb);
            Exec.Source.SqlCommand = refSql;
            Exec.Transform.ProvideInputToCache();
            LookupDF.SrcConn.Name = "SrcDb";
            LookupDF.SrcConn.SetConnectionString(srv, srcDb);
            LookupDF.Transform.Meta.OutputCollection[0].ErrorRowDisposition = Microsoft.SqlServer.Dts.Pipeline.Wrapper.DTSRowDisposition.RD_IgnoreFailure;
            LookupDF.Name = "LookupDF";
            LookupDF.DestConn.ConnectionString = "demo2.txt";
        }
        public EzLookupCachePkg(Package p) : base(p) { }   
        public static implicit operator EzLookupCachePkg(Package p) { return new EzLookupCachePkg(p); }

        // Field Names in this sample are not very good because we use templates a lot here
        [STAThread]
        static void Main(string[] args)
        {
            EzLookupCachePkg p5 = new EzLookupCachePkg(Environment.MachineName, "AdventureWorks", "AdventureWorks", 
                "select * from HumanResources.EmployeeAddress");
            p5.Exec.Transform.Connection.SetIndexCols("EmployeeID", "AddressID");
            p5.LookupDF.Source.SqlCommand = "select * from HumanResources.Employee";
            p5.LookupDF.Transform.SetJoinCols("EmployeeID,EmployeeID");
            p5.LookupDF.Transform.SetPureCopyCols("AddressID");
            p5.LookupDF.Dest.DefineColumnsInCM();
            p5.Exec.Disable = true;
            p5.LookupDF.Transform.OleDbConnection = p5.Exec.SrcConn;
            p5.LookupDF.Transform.SqlCommand = p5.Exec.Source.SqlCommand;
            p5.SaveToFile("demo5.dtsx");
            p5.Execute();
            Console.Write(string.Format("Package6 executed with result {0}\n", p5.ExecutionResult));
        }

        private static string ArrayToString<T>(T[] arr)
        {
            if (arr == null)
                return "null";
            string res = string.Empty;
            foreach (T el in arr)
                res += el.ToString() + ",";
            return res;
        }
    }
}

Here is the created package:

Now the question is: what if we want to reuse the package but don’t want cache transform to be built? The answer is – we can easily disable cache dataflow:

p6.Exec.Disable = true;

Right after that our package is ready. With case of PackageBuilder – we would have to create a separate builder without cache dataflow task.

Also pay attention to how easily lookup transform can be configured:

p6.LookupDF.Transform.SetJoinCols("EmployeeID,EmployeeKey", "LoginID,LoginID");

p6.LookupDF.Transform.SetPureCopyCols("EmployeeNationalIDAlternateKey", "ParentEmployeeNationalIDAlternateKey",
"FirstName", "LastName");
This results in the following picture in lookup transform:

Cross feature packages

Another great scenario is packages with multiple transforms in a more sophisticated dataflows. Here is an example of such package:

namespace ConsoleApplication1
{
    // For EzAPI public fields means that if we assign SSIS Package to EzPackage this field needs
    // to be assigned to the corresponding object inside SSIS package. The corresponding object is an object that has the same EzName as the field name
    // in the class. If you set it to some incorrect value - package logic won't be affected as internally it stores all the Ez components, tasks and connections
    // and uses that internal list to refresh metadata.
    class EzMyPackage : EzDataFlowPackage
    {
        public EzOleDbSource Source;
        // Transforms
        public EzLookup Lookup;
        public EzSortTransform SortMatch;
        public EzSortTransform SortNoMatch;
        // Destinations
        public EzFlatFileDestination MatchDest;
        public EzFlatFileDestination NoMatchDest;
        public EzFlatFileDestination ErrorDest;
        // Connection managers
        public EzSqlOleDbCM RefConn;
        public EzSqlOleDbCM SrcConn;
        public EzFlatFileCM MatchCM;
        public EzFlatFileCM NoMatchCM;
        public EzFlatFileCM ErrorCM;

        // Provide this constructor only if you want to overload Assignment operator
        public EzMyPackage(Package p) : base(p) { }  
        // Provide assignment operator if you want to be able to Assign SSIS Package to EzPackage
        public static implicit operator EzMyPackage(Package p) { return new EzMyPackage(p); }

        public EzMyPackage() : base()
        {
            // Connection managers
            SrcConn = new EzSqlOleDbCM(this);
            SrcConn.SetConnectionString(Environment.MachineName, "AdventureWorks");
            MatchCM = new EzFlatFileCM(this);
            MatchCM.ConnectionString = "matchcm.txt";
            NoMatchCM = new EzFlatFileCM(this);
            NoMatchCM.ConnectionString = "nomatchcm.txt";
            ErrorCM = new EzFlatFileCM(this);
            ErrorCM.ConnectionString = "errorcm.txt";
            RefConn = new EzSqlOleDbCM(this);
            RefConn.SetConnectionString(Environment.MachineName, "AdventureWorks");

            // Creating Dataflow
            Source = new EzOleDbSource(DataFlow);
            Source.Connection = SrcConn;
            Source.SqlCommand = "select * from HumanResources.Employee";
            
            Lookup = new EzLookup(DataFlow);
            Lookup.AttachTo(Source);
            Lookup.OleDbConnection = RefConn;
            Lookup.SqlCommand = "select * from HumanResources.EmployeeAddress";
            Lookup.SetJoinCols("EmployeeID,EmployeeID");
            Lookup.SetPureCopyCols("AddressID");
            Lookup.NoMatchBehavor = NoMatchBehavior.SendToNoMatchOutput;
            Lookup.OutputCol("AddressID").TruncationRowDisposition = DTSRowDisposition.RD_RedirectRow;

            SortMatch = new EzSortTransform(DataFlow);
            SortMatch.AttachTo(Lookup, 0, 0);
            SortMatch.SortOrder["EmployeeID"] = 1;      // sort in ascending order
            SortMatch.SortOrder["AddressID"] = -2;      // sort in descending order

            SortNoMatch = new EzSortTransform(DataFlow);
            SortNoMatch.AttachTo(Lookup, 1, 0);
            SortNoMatch.SortOrder["EmployeeID"] = 1;      // sort in ascending order

            ErrorDest = new EzFlatFileDestination(DataFlow);
            ErrorDest.AttachTo(Lookup, 2, 0);
            ErrorDest.Connection = ErrorCM;
            ErrorDest.DefineColumnsInCM();      // configure connection manager to have all input columns defined in the resulting file

            MatchDest = new EzFlatFileDestination(DataFlow);
            MatchDest.AttachTo(SortMatch);
            MatchDest.Connection = MatchCM;
            MatchDest.DefineColumnsInCM();

            NoMatchDest = new EzFlatFileDestination(DataFlow);
            NoMatchDest.AttachTo(SortNoMatch);
            NoMatchDest.Connection = NoMatchCM;
            NoMatchDest.DefineColumnsInCM();
        }

        [STAThread]
        static void Main(string[] args)
        {
            EzMyPackage p = new EzMyPackage();
            p.Execute();
            Console.Write(string.Format("Package executed with result {0}\n", p.ExecutionResult));
        }
    }
}

The result is:

As you can see, this relatively complex package is created even without using template possibilities of EzAPI and it the code that creates package (class constructor) is only 50 lines of code. So it took 4 lines per object to create the package. The dtsx file of this package contains more than 1300 lines.

Misc scenarios

Say we want the dataflow in a previous package to be executed in a loop, but we want a number of those loops concatenated together. Something like this:

// This demo creates a package with N forloops with dataflow inside executed sequentially
namespace ConsoleApplication1
{
    class EzMyDataFlow : EzDataFlow
    {
        public EzOleDbSource Source;
        // Transforms
        public EzLookup Lookup;
        public EzSortTransform SortMatch;
        public EzSortTransform SortNoMatch;
        // Destinations
        public EzFlatFileDestination MatchDest;
        public EzFlatFileDestination NoMatchDest;
        public EzFlatFileDestination ErrorDest;
        // Connection managers
        public EzSqlOleDbCM RefConn;
        public EzSqlOleDbCM SrcConn;
        public EzFlatFileCM MatchCM;
        public EzFlatFileCM NoMatchCM;
        public EzFlatFileCM ErrorCM;

        public EzMyDataFlow(EzContainer parent, TaskHost pipe) : base(parent, pipe) { }

        public EzMyDataFlow(EzContainer parent) : base(parent)
        {
            // Connection managers
            SrcConn = new EzSqlOleDbCM(Package, "SrcConn");
            SrcConn.SetConnectionString(Environment.MachineName, "AdventureWorks");
            MatchCM = new EzFlatFileCM(Package, "MatchCM");
            MatchCM.ConnectionString = "matchcm.txt";
            NoMatchCM = new EzFlatFileCM(Package, "NoMatchCM");
            NoMatchCM.ConnectionString = "nomatchcm.txt";
            ErrorCM = new EzFlatFileCM(Package, "ErrorCM");
            ErrorCM.ConnectionString = "errorcm.txt";
            RefConn = new EzSqlOleDbCM(Package, "RefConn");
            RefConn.SetConnectionString(Environment.MachineName, "AdventureWorks");

            // Creating Dataflow
            Source = new EzOleDbSource(this);
            Source.Connection = SrcConn;
            Source.SqlCommand = "select * from HumanResources.Employee";

            Lookup = new EzLookup(this);
            Lookup.AttachTo(Source);
            Lookup.OleDbConnection = RefConn;
            Lookup.SqlCommand = "select * from HumanResources.EmployeeAddress";
            Lookup.SetJoinCols("EmployeeID,EmployeeID");
            Lookup.SetPureCopyCols("AddressID");
            Lookup.NoMatchBehavor = NoMatchBehavior.SendToNoMatchOutput;
            Lookup.OutputCol("AddressID").TruncationRowDisposition = DTSRowDisposition.RD_RedirectRow;

            SortMatch = new EzSortTransform(this);
            SortMatch.AttachTo(Lookup, 0, 0);
            SortMatch.SortOrder["EmployeeID"] = 1;      // sort in ascending order
            SortMatch.SortOrder["AddressID"] = -2;     // sort in descending order

            SortNoMatch = new EzSortTransform(this);
            SortNoMatch.AttachTo(Lookup, 1, 0);
            SortNoMatch.SortOrder["EmployeeID"] = 1;      // sort in ascending order

            ErrorDest = new EzFlatFileDestination(this);
            ErrorDest.AttachTo(Lookup, 2, 0);
            ErrorDest.Connection = ErrorCM;
            ErrorDest.DefineColumnsInCM();      // configure connection manager to have all input columns defined in the resulting file

            MatchDest = new EzFlatFileDestination(this);
            MatchDest.AttachTo(SortMatch);
            MatchDest.Connection = MatchCM;
            MatchDest.DefineColumnsInCM();

            NoMatchDest = new EzFlatFileDestination(this);
            NoMatchDest.AttachTo(SortNoMatch);
            NoMatchDest.Connection = NoMatchCM;
            NoMatchDest.DefineColumnsInCM();
        }
    }

    public class EzMyLoopPkg : EzPackage
    {
        EzExecForLoop<EzMyDataFlow>[] Loops;
        public EzMyLoopPkg(int numLoops) : base()
        {
            Variables.Add("i", false, "", 0);
            Loops = new EzExecForLoop<EzMyDataFlow>[numLoops];
            for (int i = 0; i < numLoops; i++)
            {
                Loops[i] = new EzExecForLoop<EzMyDataFlow>(this);
                Loops[i].InitExpression = "@i=0";
                Loops[i].EvalExpression = "@i<10";
                Loops[i].AssignExpression = "@i=@i+1";
            }
            for (int i = 1; i < numLoops; i++)
                Loops[i].AttachTo(Loops[i-1]);
        }

        public static void Main(string[] args)
        {
            EzMyLoopPkg p = new EzMyLoopPkg(5);
            p.SaveToFile("demo7.dtsx");
            p.Execute();
            Console.Write(string.Format("Package executed with result {0}\n", p.ExecutionResult));
        }
    }
}

Note that in this example we reuse dataflow task in multiple loops. But only one instance of connection managers is created. This happens because EzAPI creates multiple wrappers for connection managers, but doesn’t duplicate connection managers themselves.

Extending EzAPI

Main types of objects currently supported by EzAPI are tasks, containers, packages, dataflow components, and connection managers. Let’s review how to extend EzAPI power by developing new EzObjects.

Developing Pipeline Components

The base class for all components is EzComponent. Let review how to create components using OleDBCommand transform as example. OleDBCommand has a lot of common with destination type of transforms. It requires mapping of input column to external metadata columns. That’s why we’ll use EzAdapter class as a base class. EzAdapter is a base class for all source and destination adapters.

// This attribute is mandatory. It should return GUID of the native component, of full type name 
// of the  managed component
[CompID("{8E61C8F6-C91D-43B6-97EB-3423C06571CC}")]
public class EzOleDbCommand : EzAdapter
{
    // These two constructors are mandatory. The first one creates new component in SSIS dataflow
    // The second one creates wrapper for the existing one
    public EzOleDbCommand(EzDataFlow dataFlow) : base(dataFlow) { }
    public EzOleDbCommand(EzDataFlow parent, IDTSComponentMetaData100 meta) : base(parent, meta) { }

    // The third thing that is necessary is to declare properties specific to your component. 
    // Mostly they will look like the three properties above
    public int CommandTimeout
    {
        get { return (int)m_meta.CustomPropertyCollection["CommandTimeout"].Value; }
        set { m_comp.SetComponentProperty("CommandTimeout", value); ReinitializeMetaData(); }
    }

    public int DefaultCodePage
    {
        get { return (int)m_meta.CustomPropertyCollection["DefaultCodePage"].Value; }
        set { m_comp.SetComponentProperty("DefaultCodePage", value); }
    }

    public string SqlCommand
    {
        get { return (string)m_meta.CustomPropertyCollection["SqlCommand"].Value; }
        set { m_comp.SetComponentProperty("SqlCommand", value); ReinitializeMetaData(); }
    }
}

Another example is DataConvert. Here is the code for it:

[CompID("{BD06A22E-BC69-4AF7-A69B-C44C2EF684BB}")]
public class EzDataConvert : EzComponent
{
    public EzDataConvert(EzDataFlow dataFlow) : base(dataFlow) { }
    public EzDataConvert(EzDataFlow parent, IDTSComponentMetaData100 meta) : base(parent, meta) { }

    // This function sets convertion if inputColumn to the column with the specified datatype
    public void Convert(string inColName, string newColName, DataType dataType, int length, int precision, int scale, int codePage)
    {
        LinkInputToOutput(inColName);
        SetOutputColumnProperty(newColName, "SourceInputColumnLineageID", InputCol(inColName).LineageID);
        SetOutputColumnDataTypeProperties(newColName, dataType, length, precision, scale, codePage);
    }

    // returns name of converted output column that corresponds to the specified input column
    public string ConvertedColumn(string inputColName)
    {
        int lineageId = InputCol(inputColName).LineageID;
        foreach (IDTSOutputColumn100 c in Meta.OutputCollection[0].OutputColumnCollection)
            if (OutputColumnPropertyExists(c.Name, "SourceInputColumnLineageID") && 
                (int)c.CustomPropertyCollection["SourceInputColumnLineageID"].Value == lineageId)
            {
                return c.Name;
            }
        return null;
    }

    // FastParse property set for the specified converted column
    protected ColumnCustomPropertyIndexer<bool> m_fastParse;
    public ColumnCustomPropertyIndexer<bool> FastParse
    {
        get
        {
            if (m_fastParse == null)
                m_fastParse = new ColumnCustomPropertyIndexer<bool>(this, "FastParse", IndexerType.Output, false);
            return m_fastParse;
        }
    }
}

This code is pretty straightforward and easy to understand if you are familiar with DataConvert component. The whole implementation took about 40 lines. Note that normally implementation of custom component simply includes exposing custom properties of component and, if necessary, input/output columns (like FastParse in DataConvert).

Developing Runtime Tasks

The base class for tasks is EzTask. To illustrate the process of task development let’s take ExecutePackage Task as an example.

// This attribute should be declared and return either Moniker, or full managed type name 
[ExecID("SSIS.ExecutePackageTask.2")]
public class EzExecPackage : EzTask
{
    // These two constructors are mandatory. The first one creates new task in SSIS control flow
    // The second one creates wrapper for the existing one
    public EzExecPackage(EzContainer parent) : base(parent) { }
    public EzExecPackage(EzContainer parent, TaskHost task) : base(parent, task) { }

    // Now declare properties. In most cases they will look similar to the properties below
    public bool ExecOutOfProcess
    {
        get { return (bool)host.Properties["ExecuteOutOfProcess"].GetValue(host); }
        set { host.Properties["ExecuteOutOfProcess"].SetValue(host, value); }
    }

    public string PackageName
    {
        get { return (string)host.Properties["PackageName"].GetValue(host); }
        set { host.Properties["PackageName"].SetValue(host, value); }
    }

    public string PackagePassword
    {
        get { return (string)host.Properties["PackagePassword"].GetValue(host); }
        set { host.Properties["PackagePassword"].SetValue(host, value); }
    }

    public string PackageID
    {
        get { return (string)host.Properties["PackageID"].GetValue(host); }
        set { host.Properties["PackageID"].SetValue(host, value); }
    }

    public string VersionID
    {
        get { return (string)host.Properties["VersionID"].GetValue(host); }
        set { host.Properties["VersionID"].SetValue(host, value); }
    }

    // We can only accept OLEDB or FILE connection managers.
    protected EzConnectionManager m_connection;
    public EzConnectionManager Connection
    {
        get { return m_connection; }
        set
        {
            if (value == null)
                throw new ArgumentNullException("value");
            if (value.CM.CreationName != "FILE" && value.CM.CreationName != "OLEDB")
                throw new IncorrectAssignException(string.Format("Cannot assign {0} connection to EzExecPackage task", value.CM.CreationName));
            (host.InnerObject as IDTSExecutePackage100).Connection = value.Name;
            m_connection = value;
        }
    }
}

In case of managed task everything looks pretty similar, except that ExecID returns full managed type name:

[ExecID("Microsoft.SqlServer.Dts.Tasks.ActiveXScriptTask.ActiveXScriptTask, Microsoft.SqlServer.ActiveXScriptTask, Version=10.0.0.0, Culture=neutral, PublicKeyToken=89845dcd8080cc91")]
public class EzActiveXScript : EzTask
{
    public EzActiveXScript(EzContainer parent) : base(parent) { EntryMethod = "Main"; ScriptingLanguage = "VBScript"; }
    public EzActiveXScript(EzContainer parent, TaskHost task) : base(parent, task) { }

    public string EntryMethod
    {
        get { return (string)host.Properties["EntryMethod"].GetValue(host); }
        set { host.Properties["EntryMethod"].SetValue(host, value); }
    }

    public string ExecValueVariable
    {
        get { return (string)host.Properties["ExecValueVariable"].GetValue(host); }
        set { host.Properties["ExecValueVariable"].SetValue(host, value); }
    }

    public string ScriptingLanguage
    {
        get { return (string)host.Properties["ScriptingLanguage"].GetValue(host); }
        set { host.Properties["ScriptingLanguage"].SetValue(host, value); }
    }

    public string ScriptText
    {
        get { return (string)host.Properties["ScriptText"].GetValue(host); }
        set { host.Properties["ScriptText"].SetValue(host, value); }
    }
}

Developing containers and packages

The base class for containers is EzContainer. Here is how EzForLoop container can be implemented:

[ExecID("STOCK:FORLOOP")]
public class EzForLoop : EzContainer
{
    public EzForLoop(EzContainer parent, DtsContainer c) : base(parent, c) { }
    public EzForLoop(EzContainer parent) : base(parent) { RecreateExecutables(); }

    public string AssignExpression
    {
        get { return (m_exec as ForLoop).AssignExpression; }
        set { (m_exec as ForLoop).AssignExpression = value; }
    }

    public string EvalExpression
    {
        get { return (m_exec as ForLoop).EvalExpression; }
        set { (m_exec as ForLoop).EvalExpression = value; }
    }

    public string InitExpression
    {
        get { return (m_exec as ForLoop).InitExpression; }
        set { (m_exec as ForLoop).InitExpression = value; }
    }
}

All packages derive from EzPackage class, which derives from EzContainer. Here is how a package with dataflow task can be created:

public class EzDataFlowPackage : EzPackage
{
    public EzDataFlow DataFlow;

    public EzDataFlowPackage() : base() { DataFlow = new EzDataFlow(this); }
    public EzDataFlowPackage(Package p) : base(p) { }

    public static implicit operator EzDataFlowPackage(Package p) { return new EzDataFlowPackage(p); }
}

Developing Connection Managers

Connection managers in EzAPI derive from EzConnectionManager class.

public enum FileUsageType : int
{
    ExistingFile = 0,
    CreateFile = 1,
    ExistingFolder = 2,
    CreateFolder = 3
}

[ConnMgrID("FILE")]
public class EzFileCM: EzConnectionManager
{
    public EzFileCM(EzPackage parent) : base(parent) { }
    public EzFileCM(EzPackage parent, ConnectionManager c) : base(parent, c) { }
    public EzFileCM(EzPackage parent, string name) : base(parent, name) { }

    public string DataSourceID
    {
        get { return (string)m_conn.Properties["DataSourceID"].GetValue(m_conn); }
        set { m_conn.Properties["DataSourceID"].SetValue(m_conn, value); Parent.ReinitializeMetaData(); }
    }

    public FileUsageType FileUsageType
    {
        get { return (FileUsageType)m_conn.Properties["FileUsageType"].GetValue(m_conn); }
        set { m_conn.Properties["FileUsageType"].SetValue(m_conn, value); }
    }
}

Implemented SSIS objects

Tasks and Containers

For Loop container
DataFlow Task
Execute Package Task
ActiveX Script Task

DataFlow Components

OLEDB Source
OLEDB Destination
FlatFile Source
FlatFile Destination
ADO.Net Source
ADO.Net Destination
Multicast Transform
Derived Column Transform
Sort Transform
OLEDB command Transform
Lookup Transform
Cache Transform
Data Convert Transform
Aggregate Transform

Connection Managers

OLEDB Connection manager (including specific versions for SQL Server, Oracle, DB2)
FILE connection manager
FLATFILE connection manager
CACHE connection manager
ADO.NET connection manager

Using template collections

Currently EzAPI framework includes a number of classes that can make development of packages even easier.

public class EzDataFlowPackage : EzPackage

Package with single dataflow task without any connection managers

public class EzForLoopDFPackage : EzForLoopPackage

Package with single ForLoop container with Dataflow task in it without any connection managers

public class EzSrcPackage<SrcType, SrcConnType> : EzDataFlowPackage
    where SrcType : EzAdapter
    where SrcConnType : EzConnectionManager

Package with single dataflow task that contains only source of SrcType and connection manager of type SrcConnType used by this source

public class EzSrcDestPackage<SrcType, SrcConnType, DestType, DestConnType> : EzSrcPackage<SrcType, SrcConnType>
    where SrcType : EzAdapter
    where SrcConnType : EzConnectionManager
    where DestType : EzAdapter
    where DestConnType : EzConnectionManager

Package with single dataflow task that contains source->destination dataflow and their connection managers.

public class EzTransformPackage<SrcType, SrcConnType, TransType, DestType, DestConnType>
    : EzSrcPackage<SrcType, SrcConnType>
    where SrcType : EzAdapter
    where SrcConnType : EzConnectionManager
    where TransType : EzComponent
    where DestType : EzAdapter
    where DestConnType : EzConnectionManager

Package with single dataflow task that contains source->transform->destination and connection managers for source and destination.

public class EzLoopTransformPackage<SrcType, SrcConnType, TransType, DestType, DestConnType>
    : EzForLoopDFPackage
    where SrcType : EzAdapter
    where SrcConnType : EzConnectionManager
    where TransType : EzComponent
    where DestType : EzAdapter
    where DestConnType : EzConnectionManager

Package with single dataflow task that contains source->transform->destination and connection managers for source and destination.

public class EzPkgWithExec<T> : EzPackage where T : EzExecutable

Package with a single executable (task or container)

public class EzExecForLoop<T> : EzForLoop where T : EzExecutable

Package with a single executable (task or container) in a for loop container

public class EzSrcDF<SrcComp> : EzDataFlow where SrcComp : EzComponent

Dataflow with some source component

public class EzSrcConnDF<SrcComp, SrcCM> : EzSrcDF<SrcComp>
    where SrcComp : EzAdapter
    where SrcCM : EzConnectionManager

Dataflow with some source component with connection manager

public class EzTransformDF<SrcComp, SrcCM, Trans> : EzSrcConnDF<SrcComp, SrcCM>
    where SrcComp : EzAdapter
    where SrcCM : EzConnectionManager
    where Trans : EzComponent

public class EzTransDestDF<SrcComp, SrcCM, Trans, DestComp> : EzTransformDF<SrcComp, SrcCM, Trans>
    where SrcComp : EzAdapter
    where SrcCM : EzConnectionManager
    where Trans : EzComponent
    where DestComp : EzComponent

Dataflow with some source component with connection manager, transformation and destination without connection manager

public class EzTransDestConnDF<SrcComp, SrcCM, Trans, DestComp, DestCM> : EzTransDestDF<SrcComp, SrcCM, Trans, DestComp>
    where SrcComp : EzAdapter
    where SrcCM : EzConnectionManager
    where Trans : EzComponent
    where DestComp : EzAdapter
    where DestCM : EzConnectionManager

Dataflow with some source component with connection manager, transformation and destination with connection manager.

API Sample - Create a package with a data flow task

mmasson — Wed, 31 Dec 2008 00:01:18 GMT

This code creates a package with a single data flow task. It will be used as a base for the more complicated code samples.

static void Main(string[] args)
{
    Package package = new Package();

    // Add Data Flow Task
    Executable dataFlowTask = package.Executables.Add("STOCK:PipelineTask");

    // Set the name (otherwise it will be a random GUID value)
    TaskHost taskHost = dataFlowTask as TaskHost;
    taskHost.Name = "Data Flow Task";

    // We need a reference to the InnerObject to add items to the data flow
    MainPipe pipeline = taskHost.InnerObject as MainPipe;
}

New SSIS community samples

mmasson — Tue, 19 Aug 2008 19:56:22 GMT

Some new samples have been added to the SSIS community samples project on codeplex.

Delimited Flat File Reader

This data flow source component is capable of parsing delimited flat files, including files with rows that are missing columns.

Package Generation Sample

This standalone executable shows how to use the SSIS object model to do schema-based dynamic package generation. It works with SQL Server, Excel and Flat Files as source / destinations.

Hello World Task Sample

This is the sample that I previously blogged about. It’s a simple Hello World type task that shows how to extend the base task UI classes.

SharePoint List Adapters

This sample contains data flow source and destination components that demonstrate how to get data in and out of SharePoint lists. I’ve seen a lot of requests for this one!

Note, all of the samples are available as binaries, and as source. We’re going to continue adding to the list, so if you have any ideas of what you’d like to see, feel free to send them to me.

Regular Expression Flat File Source

mmasson — Thu, 24 Jul 2008 06:52:00 GMT

The RegEx flat file source is one of the new community samples for SQL Server 2008 we’ve published to Codeplex. It uses regular expressions to extract values from a text file. It works similar to the flat file source, except that it’s not limited to CSV-type files (I saw a demo where the developer who created the sample used it to extract class names from a source files).

Note, when I ran the installer for the sample, it didn’t put the RegExFlatFileSource.dll file under 100\DTS\PipelineComponents. It only put the source down. We’ll either update the installer, or update the docs on the site. To get the component, I opened the project in Visual Studio, and ran a build. The project has a post build step which places the DLL under the PipelineComponents directory, and runs gacutil.exe to place it in the GAC.

Once the sample is installed, you can add it to your toolbar in Visual Studio.

We’ll start off with a simple flat file example. My data looks like this:

value 1,1,2001-01-01
value 2,2,2002-02-02
value 3,3,2003-03-03
value 4,4,2004-04-04
value 5,5,2005-05-05
value 6,6,2006-06-06
value 7,7,2007-07-07
value 8,8,2008-08-08

Columns are defined using groups. If I want the component to behave like a regular CSV parser, I can use a regular expression like this:

(\w+),(\w+),(\w+)

If I click on the Column Mappings tab, I can see four columns have been defined.

Like with regular expression matches, the first match (column 0) is the entire pattern (which is the whole line in this case). The next three columns are the groups I defined in my regex.

I can also provide default names for my columns by naming the groups. This regex is a little more specific, and adds names to the groups using the ?<name> syntax supported by .NET.

(?<text>.*?),(?<number>\d),(?<date>\d{4}-\d{2}-\d{2})

If I look back at the Column Mappings tab, I can see the columns now have names.

You’ll notice that the component provides two outputs – one for rows that match the pattern, and one for rows that don’t match. The non-matching row output will always have a single string column which contains the entire line.

Adding a data viewer to the Match output, I can see all of the column matches it made:

I really like this sample because it opens up a lot of data sources, like log files, that used to require custom parsing using a script component. I think it has a lot of potential!

First two community samples for 2008 now on Codeplex

mmasson — Thu, 24 Jul 2008 01:34:43 GMT

The Integration Services Community Samples project is now active on Codeplex. These samples are being created by the SSIS product team to supplement the content in books online, and the Product Samples for 2008. They will (hopefully) provide useful functionality that you can then take an customize for your ETL solutions.

The first two samples are pipeline components. We have an XML Destination, which takes one or more input streams and outputs XML documents, and a RegEx Flat File Source, which allows you to parse flat files using regular expressions.

From the releases page:

XML Destination

This sample includes source and binary for a simple XML Destination pipeline component. Use this sample to learn more about how to:

Create custom data flow destination components for use with SSIS
Build component user interfaces
Support multiple inputs on a single component

Regular Expression Flat File Source

This sample includes source and binary for a regular expression based flat file parsing source. Use this sample to learn more about how to:

Create custom data flow sources components for use with SSIS
Support multiple outputs from a single component
Define output columns
Validate metadata

These components are pretty easy to use, but I’ll follow up with a couple of posts (with screen shots!!@) that show how to use them.

Creating a custom task with a default UI

mmasson — Sat, 19 Jul 2008 04:20:05 GMT

The Developing a User Interface for a Custom Task entry in Books Online describes how to build your UI from scratch, but there is an easier way to create a UI that has the same look and feel as the stock tasks that ship with SSIS.

The Microsoft.DataTransformationServices.Controls assembly exposes a base class that most of the stock SSIS tasks inherit from – DTSBaseTaskUI. With minimal coding, you too can have a beautiful Task UI (that looks something like this:)

After creating your task, you can link it to a UI assembly using the UITypeName parameter of the DtsTask property. The recommended practice is to create a separate assembly for your UI to separate the UI code from the code that gets loaded at runtime, but this is optional. It can all work in the same assembly as well.

We’re going to create three classes for our “HelloWorldTask”

HelloWorldTaskUI – Implements IDtsTaskUI
HelloWorldTaskMainWnd – Extends DTSBaseTaskUI
GeneralView – Implements IDTSTaskUIView

HelloWorldTaskUI is the entry point for the UI functionality, and is responsible for initializing and displaying your UI.

Extending DTSBaseTaskUI in HelloWorldTaskMainWnd gives you the form in the screen shot above. Each page of properties you want to display is a separate class that implements IDTSTaskUIView. You get the “Expressions” page for free.

Let’s walk through what you need to do.

First you’ll need to add some additional references:

Microsoft.DataTransformationServices.Controls
Microsoft.SqlServer.Dts.Design
System.Drawing
System.Windows.Forms

Note, that Microsoft.DataTransformationServices.Controls.dll is in the GAC, but doesn’t show up in the “.NET” tab of the “Add Reference…” dialog.

Add new User Control – GeneralView.cs. On the designer, add a property grid. To keep the same look and feel as other SSIS tasks, the InitializeComponents() should look something like this:

// 
// propertyGrid
// 
this.propertyGrid.Dock = System.Windows.Forms.DockStyle.Fill;
this.propertyGrid.Location = new System.Drawing.Point(0, 0);
this.propertyGrid.Name = "propertyGrid";
this.propertyGrid.PropertySort = System.Windows.Forms.PropertySort.Categorized;
this.propertyGrid.Size = new System.Drawing.Size(150, 150);
this.propertyGrid.TabIndex = 0;
this.propertyGrid.ToolbarVisible = false;

In your GeneralView class, define an internal class which contains all of the properties you’d like to display in the UI – ie. GeneralViewNode

internal class GeneralViewNode
{
    // Properties variables
    private string displayText = string.Empty;
    private string name = string.Empty;
    private string description = string.Empty;

    internal TaskHost taskHost = null;
    internal IDtsConnectionService connectionService = null;

    internal GeneralViewNode(TaskHost taskHost, IDtsConnectionService connectionService)
    {
        this.taskHost = taskHost;
        this.connectionService = connectionService;

        // Extract common values from the Task Host
        name = taskHost.Name;
        description = taskHost.Description;

        // Extract values from the task object
        HelloWorldTask.HelloWorldTask task = taskHost.InnerObject as HelloWorldTask.HelloWorldTask;
        if (task == null)
        {
            string msg = string.Format("Type mismatch for taskHost inner object.");
            throw new ArgumentException(msg);
        }

        displayText = task.DisplayText;
    }

    #region Properties

    [Category("General"), Description("Task name")]
    public string Name
    {
        get
        {
            return name;
        }
        set
        {
            string v = value.Trim();
            if (string.IsNullOrEmpty(v))
            {
                throw new ApplicationException("Task name cannot be empty");
            }
            name = v;
        }
    }

    [Category("General"), Description("Task description")]
    public string Description
    {
        get
        {
            return description;
        }
        set
        {
            description = value.Trim();
        }
    }

    [Category("General"), Description("Text to display")]
    public string DisplayText
    {
        get
        {
            return displayText;
        }
        set
        {
            displayText = value;
        }
    }

    #endregion
}

As you can see, we have the single property for the task (DisplayText), and two of the common Task properties – Name, and Description.

Create a member variable for this node class:

private GeneralViewNode generalNode;
internal GeneralViewNode GeneralNode
{
    get { return generalNode; }
}

Implement the IDTSTaskUIView interface. These methods are called when the UI is opened, and when the UI clicks OK.

#region IDTSTaskUIView Members

public void OnCommit(object taskHost)
{
    TaskHost host = taskHost as TaskHost;
    if (host == null)
        throw new ArgumentException("Arugment is not a TaskHost.", "taskHost");

    HelloWorldTask.HelloWorldTask task = host.InnerObject as HelloWorldTask.HelloWorldTask;
    if (task == null)
    {
        throw new ArgumentException("Arugment is not a HelloWorldTask.", "taskHost");
    }

    host.Name = generalNode.Name;
    host.Description = generalNode.Description;

    // Task properties
    task.DisplayText = generalNode.DisplayText;
}

public void OnInitialize(IDTSTaskUIHost treeHost, TreeNode viewNode, object taskHost, object connections)
{
    this.generalNode = new GeneralViewNode(taskHost as TaskHost, connections as IDtsConnectionService);
    this.propertyGrid.SelectedObject = generalNode;
}

public void OnLoseSelection(ref bool bCanLeaveView, ref string reason)
{
}

public void OnSelection()
{
}

public void OnValidate(ref bool bViewIsValid, ref string reason)
{
}

#endregion

Add new Windows Form – HelloWorldTaskMainWnd.cs. Change the inheritance from Form to DTSBaseTaskUI.

Note, after inheriting from DTSBaseTaskUI, you’ll get an error when trying to open the code in Design view in Visual Studio 2008. This is caused by DTSBaseTaskUI not having a default constructor. We’re aware of the issue, and will hopefully be able to resolve it in a cumulative update. It won’t cause issues at runtime.

Add some members for the UI properties we’ll use to initialize DTSBaseTaskUI.

private const string Title = "Hello World Task";
private const string Description = "Displays a message box";
private static Icon TaskIcon = new Icon(typeof(HelloWorldTask), "Task.ico");

Add a member for GeneralView class.

private GeneralView generalView;
public GeneralView GeneralView
{
    get { return generalView; }
}

Add a constructor.

public HelloWorldTaskUIMainWnd(TaskHost taskHost, object connections) :
    base(Title, TaskIcon, Description, taskHost, connections)
{
    InitializeComponent();
    
    // Setup our views
    generalView = new GeneralView();
    this.DTSTaskUIHost.FastLoad = false;
    this.DTSTaskUIHost.AddView("General", generalView, null);
    this.DTSTaskUIHost.FastLoad = true;
}

Note, the first parameter you use for AddView() is the text that gets displayed on the left of the main window.

Finally, add a new class that implements IDtsTaskUI to control the launching of your new window.

public class HelloWorldTaskUI : IDtsTaskUI
{
    private TaskHost taskHost = null;
    private IDtsConnectionService connectionService = null;

    #region IDtsTaskUI Members

    public void Delete(IWin32Window parentWindow)
    {
    }

    public ContainerControl GetView()
    {
        return new HelloWorldTaskUIMainWnd(taskHost, connectionService);
    }

    public void Initialize(TaskHost taskHost, IServiceProvider serviceProvider)
    {
        this.taskHost = taskHost;
        this.connectionService = serviceProvider.GetService(typeof(IDtsConnectionService)) as IDtsConnectionService;
    }

    public void New(IWin32Window parentWindow)
    {
    }

    #endregion
}

And there you go! Be sure to add the task UI assembly to the GAC before trying it out in Visual Studio.

I should mention that Kirk’s book has a chapter on creating custom tasks that gives a lot more details than I have here. If you’re interested in extending the SSIS platform, Kirk’s book (or the “SSIS White Book” as I like to call it) is an invaluable resource.

I’ve uploaded the source for the highly useful Hello World Task (and its UI) to my Sky Drive public folder. I developed this using SQL Server 2008, but it should also work the same way in 2005.

I’ll also be turning this into a community sample to be published on Codeplex (along with our other new development samples) sometime soon.

Enjoy!

MDDE appears on Codeplex

mmasson — Sat, 22 Mar 2008 08:38:22 GMT

The SQL Server Metadata-Driven ETL Studio (MDDE) was an internal MSIT project first shown at the MS BI Conference last year. The initial release is now up on Codeplex. There isn't much information up about it yet, but I'm told that docs and examples should be appearing soon.

Essentially what MDDE gives you is a way to create SSIS package templates, which can be easily updated when your metadata changes. I'm currently playing around the code, and will hopefully have a walk-through up in the next little while.

Real world scenarios

mmasson — Fri, 01 Jun 2007 20:14:00 GMT

May has been a busy month!

The first Microsoft Business Intelligence conference was earlier in the month, and it seemed to be a big success. I was amazed at how many people were there, and I met a lot of interesting people when I was manning the demo booth and Ask The Experts table we had setup in one of the pavilions. I got to see some interesting talks, and previews of the new BI offerings MS has in the works (PerformancePoint Server 2007 looks very slick).

The SSIS team has been really busy working on Katmai, and we're now approaching our first public CTP. Typically before each release we hold an internal "Bug Bash", where the whole team spends the day running through user scenarios trying to find any last minute issues. This time we decided to do something a bit more involved...

The group was split up into various teams, and each team was given a real world scenario that we had to solve using SSIS. The idea was to give the product a workout, as well as give us a better point of view on some of the customer pain points. We were given a limited amount of time, so most solutions probably won't have full error handling, logging or other fail safe mechanisms, such as transactions and checkpoints. We hope to publish the scenarios and packages to Codeplex however, and continue adding to them until they can be used as "Best Practice" type examples for common problems.

Over the next little while I'll post some of the projects and go over the thought process behind the solutions.

Roll your own Transfer SQL Server Objects task

mmasson — Thu, 19 Apr 2007 02:08:00 GMT

Awhile back we went through a lot of customer feedback logged through the Connect site concerning the Transfer SQL Server Objects task. Some of these turned out to be bugs, but a lot of the reported issues stemmed from usability problems with the task itself. There's actually not that much to the task on the SSIS end – it uses the SQL Server Management Objects (SMO) API to do the bulk of the work. Anyone familiar with the SMO Transfer object will notice the similarities in the task's UI. We end up exposing the majority of the Transfer object's members as properties, which leads to most of the reported problems with the task; it's complicated, and options sometimes conflict with each other.

It's always a challenge to get the right balance of usability and functionality, and we're hoping to address the problems with this task in a future release (suggestions are always welcome). In the mean time, you can always "roll your own" version of task using the SMO API directly through a script task.

First you'll need to add references to the three SMO assemblies (found under Microsoft SQL Server\90\SDK\Assemblies):

Microsoft.SqlServer.ConnectionInfo
Microsoft.SqlServer.Smo
Microsoft.SqlServer.SmoEnum

Import the SMO namespaces in your code:

Imports Microsoft.SqlServer.Management.Smo

Imports Microsoft.SqlServer.Management.Common

The following code was adapted from one of the samples on MSDN. It creates a new database on your local server, and sets up a Transfer object to copy across all tables, dependencies, and data from the AdventureWorks database.

Public Sub Main()

Dim dbSourceName As String = "AdventureWorks"

Dim dbDestName As String = dbSourceName + "Copy"

'Connect to the local, default instance of SQL Server.

Dim srv As Server

srv = New Server

'Reference the source database

Dim db As Database

db = srv.Databases(dbSourceName)

'Create a new database that is to be destination database.

Dim dbCopy As Database

dbCopy = New Database(srv, dbDestName)

dbCopy.Create()

'Define a Transfer object and set the required options.

Dim xfr As Transfer

xfr = New Transfer(db)

xfr.CopyAllTables = True

xfr.Options.WithDependencies = True

xfr.Options.ContinueScriptingOnError = True

xfr.DestinationDatabase = dbCopy.Name

xfr.DestinationServer = srv.Name

xfr.DestinationLoginSecure = True

xfr.CopySchema = True

'Include data

xfr.CopyData = True

'Execute the transfer

xfr.TransferData()

Dts.TaskResult = Dts.Results.Success

End Sub

Another useful method of the Transfer object is ScriptTransfer() – it returns the T-SQL script of what you've configured your Transfer object to do. You can dump this to debug any issues you run into, and to figure out exactly what SMO is doing.

Interestingly, SMO actually uses SSIS internally to do its data transfer. It creates a couple of temporary packages on-the-fly, and runs behind the scenes. (It's possible to capture the scripts and packages if you set break points in the right place – send me an email or leave comments if you're interested in how to do this).

SQL Server on Codeplex

mmasson — Fri, 30 Mar 2007 04:15:00 GMT

SQL Server now has its own section on codeplex (Microsoft's open source project hosting site). This site includes all of the samples that come with SP2, in a single-serve format. Each product has a section, so you can grab samples for individual products instead of running the monolithic samples installer. These samples should be updated regularly, and will hopefully grow with contributions from our MVPs.

As we near our first public technical preview, you can expect to see some Katmai samples showing up as well!