September 10, 2015

Connect To PostgreSQL Using SSIS

Lately I have been working on a sync process between postgres and other databases, so after much research about drivers I found that the odbc drivers for postgres really sucks. So, after much pain I found the NPSQL drivers and after some thought said wait... I can connect using the script task and generate the files of my tables using metadata and because is ssis not need to build a new service. It is a pain but it gets the job done.
So... Here it is...
/*
   Microsoft SQL Server Integration Services Script Task
   Write scripts using Microsoft Visual C# 2008.
   The ScriptMain is the entry point class of the script.
*/

using System;
using Npgsql;
using System.Data;
using System.IO;
using System.Text;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;

namespace ST_b7b6f2660bba404bb3315fc71b301f0e.csproj
{
    [System.AddIn.AddIn("ScriptMain", Version = "1.0", Publisher = "", Description = "")]
    public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
    {

        #region VSTA generated code
        enum ScriptResults
        {
            Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
            Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
        };
        #endregion

        /*
The execution engine calls this method when the task executes.
To access the object model, use the Dts property. Connections, variables, events,
and logging features are available as members of the Dts property as shown in the following examples.

To reference a variable, call Dts.Variables["MyCaseSensitiveVariableName"].Value;
To post a log entry, call Dts.Log("This is my log text", 999, null);
To fire an event, call Dts.Events.FireInformation(99, "test", "hit the help message", "", 0, true);

To use the connections collection use something like the following:
ConnectionManager cm = Dts.Connections.Add("OLEDB");
cm.ConnectionString = "Data Source=localhost;Initial Catalog=AdventureWorks;Provider=SQLNCLI10;Integrated Security=SSPI;Auto Translate=False;";

Before returning from this method, set the value of Dts.TaskResult to indicate success or failure.

To open Help, press F1.
*/
        Variables vars = null;
        public void Main()
        {
           
            
            //SSIS Variables 
            Dts.VariableDispenser.LockForRead("User::strBatchId");
            Dts.VariableDispenser.LockForWrite("User::strFileName");
            Dts.VariableDispenser.LockForRead("User::strTableName");
            Dts.VariableDispenser.LockForRead("User::strSqlCmd");
            Dts.VariableDispenser.LockForRead("User::strFileDirectoryPath");
            Dts.VariableDispenser.LockForRead("User::strPostgresConStr");

            Dts.VariableDispenser.GetVariables(ref vars);
            String FileName = vars["User::strTableName"].Value.ToString() + "_" + vars["User::strBatchId"].Value.ToString()+".txt";
       

            String Delimiter = "|";
            int PromoBatchId = 5;
            String fullFilePath = vars["User::strFileDirectoryPath"].Value.ToString() + FileName;
            Boolean isSuccess;
            //String UID = "postgres";
            //String PWD = "Pa$$w0rd1";
            String sqlcmd = vars["User::strSqlCmd"].Value.ToString();

            String ConnStr = vars["User::strPostgresConStr"].Value.ToString();
                //String.Format("Pooling=true;MinPoolSize=1;MaxPoolSize=20;Timeout=30;Server=" + ServerName + ";Database=" + DatabaseName + ";Port=" + DbPort + ";Integrated Security=true");

            isSuccess = Generate_PostGres_File(ConnStr, sqlcmd, fullFilePath, PromoBatchId,Delimiter);
            vars.Unlock();

            if (isSuccess)
                Dts.TaskResult = (int)ScriptResults.Success;
            else
                Dts.TaskResult = (int)ScriptResults.Failure;


            // TODO: Add your code here
            
        }
        /*
         * Get Header File
         */
        private String GetFileHeader(ref DataTable dt,String FileDelimiter)
        {
            StringBuilder HeaderFile = new StringBuilder();

            String qoute = "";
            String RS = System.Environment.NewLine;
            foreach(DataColumn col in dt.Columns)
            {
                HeaderFile.Append(FileDelimiter);
                HeaderFile.Append(qoute + col.ColumnName + qoute);
               
            }
            HeaderFile.Remove(0,1);       
          
            HeaderFile.Append(RS);
           return HeaderFile.ToString();
        }
        private Boolean Generate_PostGres_File(String ConnectionString, String SqlCmd,String FullFilePath,int PromoBatchId,String FileDelimiter)
        {
            String tablename = vars["User::strTableName"].Value.ToString();
            Boolean head = false;
            String qoute = "";
            String RS = System.Environment.NewLine;
            try
            {
                using (StreamWriter sw = File.CreateText(FullFilePath))
                {
                    using (NpgsqlConnection conn = new NpgsqlConnection(ConnectionString))
                    {
                      
                        conn.Open();
                        using (NpgsqlCommand cmd = new NpgsqlCommand(SqlCmd, conn))
                        {
                            //if (tablename.ToLower().CompareTo("promotion")==0)
                            //{
                            //    cmd.Parameters.AddWithValue("promobatchid", PromoBatchId);
                            //}
                            //cmd.CommandText = sqlcmd;
                            using (NpgsqlDataReader reader = cmd.ExecuteReader())
                            {
                                DataTable dt = new DataTable();
                               
                                dt.Load(reader);
                                String header = GetFileHeader(ref dt, FileDelimiter);
                                foreach (DataRow row in dt.Rows)
                                {
                                    StringBuilder rdata = new StringBuilder();
                                    StringBuilder tdata = new StringBuilder();
                                    if (head == false)
                                    {
                                        rdata.Append(header);
                                    }
                                    head = true;
                                    foreach (object item in row.ItemArray)
                                    {
                                      if (item is System.DBNull)
                                      {
                                          tdata.Append(FileDelimiter + qoute + qoute);
                                      }
                                    
                                    else
                                    {
                                        String columnData = item.ToString();
                                        tdata.Append(FileDelimiter);
                                        tdata.Append(qoute + columnData + qoute);
                                        
                                    }
                                     
                                      
                                    }
                                    tdata.Remove(0, 1);
                                    tdata.Append(RS);
                                    rdata.Append(tdata.ToString());
                                    tdata = null;

                                    sw.Write(rdata);
                                    rdata = null;
                                   
                              
                                }

                            }
                        }
                       
                        conn.Clone();
                    }
                }
                 return true;
            }
               
            catch (Exception e) 
            {
                Dts.Events.FireError(0, "PostGres FileGenerator Error", e.ToString(), string.Empty, 0);
                return false;
              
            }
        }
    }
}

7 comments:

  1. Excellent article this resumes all issues that I had during the SSIS Postgresql Write and I was looking in several posts on the web but here summarizes perfectly.

    ReplyDelete
  2. Very helpful. Very clear! Thanks!I have been searching for a solution to my SSIS Postgresql Write issue with an SSIS project for weeks now. Thanks to you I finally found my solution.

    ReplyDelete
  3. I think there is definitely a need to learn more about SSIS Postgre.

    SSIS Postgresql Read

    ReplyDelete
  4. This comment has been removed by the author.

    ReplyDelete
  5. This comment has been removed by the author.

    ReplyDelete
  6. Hi Jorge Novo! Thanks for this post, i was looking for something similar. Could you point me to any help on How to install the driver in my SSIS? I tried to add the npgsql package to my ScriptTask but it doesn't stick, it disappears after i close the task... Thank you

    ReplyDelete

Contact Form

Name

Email *

Message *