June 28, 2017

PostgreSQL Quarterly Partition Function

Here is some quick code on how to create a partition function using inherit tables on PostgreSql, is simple but hopefully this will save some poor soul the documentation reading and for me save me some time in the future, currently things been very busy working in Ms Sql and PostgreSql doing several conversions from one DB to the other.
1a.-Create sequence
1b.- Create table
2.-Create function
3.-create trigger




-- FUNCTION:
 TableSchemaNameHere.TableName_partition_insert()

-- DROP FUNCTION TableSchemaNameHere.event_partiont_insert();
CREATE SEQUENCE seq_TableSchemaNameHere.TableNameHere_TableColumn
minvalue 1 nomaxvalue increment by 1;

create table TableSchemaNameHere.Tablename
(
 PKID int  DEFAULT nextval('seq_TableSchemaNameHere.TableNameHere_TableColumn') NOT NULL,
 insertdate timestamp(6) without time zone default now(),
 time timestamp without time zone,
 constraint pk_ primary key (pkid)
)

CREATE or Replace FUNCTION TableSchemaNameHere.TableName_partiont_insert()
    RETURNS trigger
    LANGUAGE 'plpgsql'
    COST 100.0
    VOLATILE NOT LEAKPROOF
   
AS $BODY$

                              declare
                                             declare
                                             _partition_date varchar(50);
               _partition varchar(80);
            _IsNewChild boolean;
             _current_day int;
            _first_day varchar(2) := '01';
            _last_day varchar(2);
            _pick_day varchar(2);

    begin
               -- get current day
        _current_day  := date_part('day',new.time);
     
        -- find last day of the month function
         _last_day := date_part ('day',TableSchemaNameHere.last_day (new.time::date));
        -- get YYYY_MM format
               _partition_date := to_char (NEW.time,'YYYY_MM');
     
        -- find if first or second sequence
        if _current_day < 16
         then
            _pick_day := _first_day;
            _partition_date  := _partition_date||'_'|| _first_day;
         
          else
           _pick_day = _last_day;
           _partition_date := _partition_date||'_'|| _last_day;
         end if;
       
       _partition := TG_TABLE_NAME || '_' || _partition_date;
        if not exists ( select 1 from pg_tables where schemaname= TG_TABLE_SCHEMA and tablename= _partition)
        then
     
               RAISE NOTICE 'A partition has been created %',TG_TABLE_SCHEMA ||'.'|| _partition;
            _IsNewChild = TRUE;
            EXECUTE 'create table '|| TG_TABLE_SCHEMA ||'.'|| _partition || ' (check( to_char (time,''YYYY_MM'||'_'|| _pick_day||''') = '''|| to_char (NEW.time,'YYYY_MM')||'_'||_pick_day||''')) INHERITS ( TableSchemaNameHere.TableNamehere);';
         
        end if;
        execute 'insert into '|| TG_TABLE_SCHEMA || '.' || _partition || ' SELECT(' || TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').* RETURNING eventid;';
        if (_IsNewChild)
         then
           -- Add primary key
                                                            EXECUTE format('ALTER TABLE '||TG_TABLE_SCHEMA||'.%s ADD PRIMARY KEY(pkid)', _partition);
                -- Assign owner of of inherited table
                                                             EXECUTE format('ALTER TABLE '||TG_TABLE_SCHEMA||'.%s OWNER TO db_owner', _partition);

                                                         
                                                            -- Add FK promo_batch_id
                                                            EXECUTE format('ALTER TABLE '||TG_TABLE_SCHEMA||'.%s ADD CONSTRAINT FK_%s FOREIGN KEY(columnId) REFERENCES TableSchemaNameHere.fkid (fkid)  MATCH FULL',_tablename, _tablename);

                                                            -- Need to define indexes for inherited tables
                                                            EXECUTE format('CREATE INDEX ix_%s_EventTypeID ON '||TG_TABLE_SCHEMA||'.%s USING btree ("indexkeycolumn")', _tablename, _tablename);
             
                                                         
         end if;
        RETURN NULL;
    end
 
$BODY$;

----Create trigger
CREATE TRIGGER tr_TableName_Partition
BEFORE INSERT ON TableSchemaNameHere.TableNameHere
FOR EACH ROW EXECUTE PROCEDURE myschema.server_partition_function();

May 31, 2017

Get objects from AWS S3

Continue to work on pulling and pushing objects into S3 aws so here is a quick and dirty code cannibalize around...

 #region Help:  Introduction to the script task
/* The Script Task allows you to perform virtually any operation that can be accomplished in
* a .Net application within the context of an Integration Services control flow.
 *
 * Expand the other regions which have "Help" prefixes for examples of specific ways to use
* Integration Services features within this script task. */
#endregion


#region Namespaces
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.IO;
using Amazon.S3;
using Amazon.S3.Model;
using Amazon;
#endregion

namespace ST_951ab847fc194587b642a51853cb7536
{
    /// <summary>
    /// ScriptMain is the entry point class of the script.  Do not change the name, attributes,
    /// or parent of this class.
    /// </summary>
       [Microsoft.SqlServer.Dts.Tasks.ScriptTask.SSISScriptTaskEntryPointAttribute]
       public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
       {
       #region Help:  Using Integration Services variables and parameters in a script
        /* To use a variable in this script, first ensure that the variable has been added to
         * either the list contained in the ReadOnlyVariables property or the list contained in
         * the ReadWriteVariables property of this script task, according to whether or not your
         * code needs to write to the variable.  To add the variable, save this script, close this instance of
         * Visual Studio, and update the ReadOnlyVariables and
         * ReadWriteVariables properties in the Script Transformation Editor window.
         * To use a parameter in this script, follow the same steps. Parameters are always read-only.
         *
         * Example of reading from a variable:
         *  DateTime startTime = (DateTime) Dts.Variables["System::StartTime"].Value;
         *
         * Example of writing to a variable:
         *  Dts.Variables["User::myStringVariable"].Value = "new value";
         *
         * Example of reading from a package parameter:
         *  int batchId = (int) Dts.Variables["$Package::batchId"].Value;
         * 
         * Example of reading from a project parameter:
         *  int batchId = (int) Dts.Variables["$Project::batchId"].Value;
         *
         * Example of reading from a sensitive project parameter:
         *  int batchId = (int) Dts.Variables["$Project::batchId"].GetSensitiveValue();
         * */

        #endregion

        #region Help:  Firing Integration Services events from a script
        /* This script task can fire events for logging purposes.
         *
         * Example of firing an error event:
         *  Dts.Events.FireError(18, "Process Values", "Bad value", "", 0);
         *
         * Example of firing an information event:
         *  Dts.Events.FireInformation(3, "Process Values", "Processing has started", "", 0, ref fireAgain)
         *
         * Example of firing a warning event:
         *  Dts.Events.FireWarning(14, "Process Values", "No values received for input", "", 0);
         * */
        #endregion

        #region Help:  Using Integration Services connection managers in a script
        /* Some types of connection managers can be used in this script task.  See the topic
         * "Working with Connection Managers Programatically" for details.
         *
         * Example of using an ADO.Net connection manager:
         *  object rawConnection = Dts.Connections["Sales DB"].AcquireConnection(Dts.Transaction);
         *  SqlConnection myADONETConnection = (SqlConnection)rawConnection;
         *  //Use the connection in some code here, then release the connection
         *  Dts.Connections["Sales DB"].ReleaseConnection(rawConnection);
         *
         * Example of using a File connection manager
         *  object rawConnection = Dts.Connections["Prices.zip"].AcquireConnection(Dts.Transaction);
         *  string filePath = (string)rawConnection;
         *  //Use the connection in some code here, then release the connection
         *  Dts.Connections["Prices.zip"].ReleaseConnection(rawConnection);
         * */
        #endregion


             /// <summary>
        /// This method is called when this script task executes in the control flow.
        /// Before returning from this method, set the value of Dts.TaskResult to indicate success or failure.
        /// To open Help, press F1.
        /// </summary>
             public void Main()
             {
            AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(CurrentDomain_AssemblyResolve);
                    // TODO: Add your code here
            GetBucketFiles();
                    Dts.TaskResult = (int)ScriptResults.Success;
             }
        private void GetBucketFiles()
        {
            String ak = “AmazonKey”
            String sk = "AmazonSecretKey";

            string BUCKET_NAME = @"BUCKETNAME";
             AmazonS3Client client = new AmazonS3Client(ak,sk,Amazon.RegionEndpoint.USEast1);

            // List all objects
            ListObjectsRequest listRequest = new ListObjectsRequest
            {
                BucketName = BUCKET_NAME,
                MaxKeys = 1000
              
                
            };

            ListObjectsResponse listResponse;
            do
            {
                // Get a list of objects
                listResponse = client.ListObjects(listRequest);
                foreach (S3Object obj in listResponse.S3Objects)
                {
                    if (obj.Key.EndsWith(".log"))
                    {
                        String FileName = obj.Key;
                        GetObjectRequest request = new GetObjectRequest
                        {
                            BucketName = BUCKET_NAME,
                            Key =  FileName
                        };

                        using (GetObjectResponse response = client.GetObject(request))
                        {
                            string dest = Path.Combine(@"DirectoryPath", FileName.Replace(@"DirectoryKey",""));
                            if (!File.Exists(dest))
                            {
                                response.WriteResponseStreamToFile(dest);
                            }
                          
                        }
                    }
                   
                   
                }

               } while (listResponse.IsTruncated);
        }

        #region ScriptResults declaration
        /// <summary>
        /// This enum provides a convenient shorthand within the scope of this class for setting the
        /// result of the script.
        ///
        /// This code was generated automatically.
        /// </summary>
        enum ScriptResults
        {
            Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
            Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
        };
        #endregion
        static System.Reflection.Assembly CurrentDomain_AssemblyResolve(object sender, ResolveEventArgsargs)
        {

            if (args.Name.Contains("AWSSDK.Core"))
            {
                string path = @"C:\Program Files (x86)\AWS SDK for .NET\bin\Net45\";
                return System.Reflection.Assembly.LoadFile(System.IO.Path.Combine(path,"AWSSDK.Core.dll"));
            }

            /*if (args.Name.Contains("AWSSDK.KeyManagementService"))
            {
                string path = @"C:\Program Files (x86)\AWS SDK for .NET\bin\Net45\";
                return System.Reflection.Assembly.LoadFile(System.IO.Path.Combine(path, "AWSSDK.KeyManagementService.dll"));
            }*/
            if (args.Name.Contains("AWSSDK.S3"))
            {
                string path = @"C:\Program Files (x86)\AWS SDK for .NET\bin\Net45\";
                return System.Reflection.Assembly.LoadFile(System.IO.Path.Combine(path,"AWSSDK.S3.dll"));
            }
            return null;
        }

       }
}

March 14, 2017

Streaming Data Into AWS S3 using KMS Encryption

After a few day of coding, finally I have develop a script component that can upload files into S3 aws buckets, it was a pain to do it this way, if I have a more time, next time it will be done as a Java service instead.
Cheers.





using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Security.Cryptography;
using Amazon.KeyManagementService;
using Amazon.KeyManagementService.Model;
using Amazon.S3;
using Amazon.S3.Encryption;
using Amazon.S3.Model;

namespace ConsoleEncryptStream
{
    class Program
    {
        static void Main(string[] args)
        {
            String ak = Amazonkey
            String sk = amazonSecretKey

          String arn = KMS_ALIAS;
          Stream input = new FileStream(@"C:\temp\test.txt", FileMode.Open, FileAccess.Read);
          Stream output = new FileStream(@"\\test.kms", FileMode.Create, FileAccess.Write);
        //  FileUpload(ak, sk, arn);
        //  ClientEncryption(input,ak, sk, arn);
         clientDecryption(ak, sk, arn);
         // Encrypt(input, output, ak, sk, arn);
         // Decrypt(input,output,ak,sk,arn);
        }

        static void FileUpload(string AmazonKey, string AmazonSecretKey, string arn)
        {
            try
            {

                var client = new AmazonS3Client(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1);
                string BUCKET_NAME = "BucketName";
           
               /* ListBucketsResponse response = client.ListBuckets();
                bool found = false;
                foreach (S3Bucket bucket in response.Buckets)
                {
                    if (bucket.BucketName == BUCKET_NAME)
                    {
                        found = true;
                        break;
                    }
                }
                if (found == false)
                {
                    client.PutBucket(new PutBucketRequest().BucketName = BUCKET_NAME);
                }*/
                String S3_KEY = "create/FileList.txt";
                PutObjectRequest request =( new PutObjectRequest(){
                    BucketName = BUCKET_NAME,
                  Key = S3_KEY,
                ContentBody = "This is body of S3 object.",
            });
                client.PutObject(request);
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message.ToString());
                Console.Read();
            }
        }
        static void ClientEncryption(Stream input, string AmazonKey, string AmazonSecretKey, string arn)
        {
            try
            {
                string BUCKET_NAME = "BucketName";
                using (var client = new AmazonKeyManagementServiceClient(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1))
                using (var algorithm = new KMSAlgorithm(client, arn))
                {
                    var materials = new EncryptionMaterials(algorithm);
                    var s3client = new AmazonS3EncryptionClient(AmazonKey,AmazonSecretKey,Amazon.RegionEndpoint.USEast1,materials);
  
                   PutObjectRequest request =( new PutObjectRequest()
                    {
                        BucketName = BUCKET_NAME,
                        Key = "create/outfile.txt",
                        InputStream = input,
                       // CannedACL = S3CannedACL.PublicRead,
                       
                    });
                   s3client.PutObject(request);
                   
                }
                Console.WriteLine("File Uploaded...");
                Console.Read();
                      
            }
           
            catch (Exception e)
            {
                Console.WriteLine(e.Message.ToString());
                Console.Read();
            }
           
        }
        static void clientDecryption (string AmazonKey, string AmazonSecretKey, string arn)
        {
            try
            {
               
                string BUCKET_NAME = "BucketName";
                string FileName = "create/test.txt";
                using (var client = new AmazonKeyManagementServiceClient(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1))
                using (var algorithm = new KMSAlgorithm(client, arn))
                {
                    var materials = new EncryptionMaterials(algorithm);
                    var s3client = new AmazonS3EncryptionClient(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1, materials);

                    GetObjectRequest request = (new GetObjectRequest()
                    {
                        BucketName = BUCKET_NAME,
                        Key = FileName,
                       
                        // CannedACL = S3CannedACL.PublicRead,

                    });
                    using (var response = s3client.GetObject(request))
                    {
                        string dest = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), FileName);
                        if (!File.Exists(dest))
                        {
                            response.WriteResponseStreamToFile(dest);
                        }
                    }
                   // s3client.PutObject(request);

                }
                Console.WriteLine("File downloaded...");
                Console.Read();

            }

            catch (Exception e)
            {
                Console.WriteLine(e.Message.ToString());
                Console.Read();
            }
        }
        static void Encrypt(Stream input, Stream output, string AmazonKey, string AmazonSecretKey, string arn)
        {
            try
            {
                using (var kms = new AmazonKeyManagementServiceClient(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1))
                using (var algorithm = Aes.Create())
                {
                    var dataKey = kms.GenerateDataKey(new GenerateDataKeyRequest
                    {
                        KeyId = arn,
                        KeySpec = DataKeySpec.AES_256
                    });
                    output.WriteByte((byte)dataKey.CiphertextBlob.Length);

Contact Form

Name

Email *

Message *