May 31, 2017

Get objects from AWS S3

Continue to work on pulling and pushing objects into S3 aws so here is a quick and dirty code cannibalize around...

 #region Help:  Introduction to the script task
/* The Script Task allows you to perform virtually any operation that can be accomplished in
* a .Net application within the context of an Integration Services control flow.
 *
 * Expand the other regions which have "Help" prefixes for examples of specific ways to use
* Integration Services features within this script task. */
#endregion


#region Namespaces
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.IO;
using Amazon.S3;
using Amazon.S3.Model;
using Amazon;
#endregion

namespace ST_951ab847fc194587b642a51853cb7536
{
    /// <summary>
    /// ScriptMain is the entry point class of the script.  Do not change the name, attributes,
    /// or parent of this class.
    /// </summary>
       [Microsoft.SqlServer.Dts.Tasks.ScriptTask.SSISScriptTaskEntryPointAttribute]
       public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
       {
       #region Help:  Using Integration Services variables and parameters in a script
        /* To use a variable in this script, first ensure that the variable has been added to
         * either the list contained in the ReadOnlyVariables property or the list contained in
         * the ReadWriteVariables property of this script task, according to whether or not your
         * code needs to write to the variable.  To add the variable, save this script, close this instance of
         * Visual Studio, and update the ReadOnlyVariables and
         * ReadWriteVariables properties in the Script Transformation Editor window.
         * To use a parameter in this script, follow the same steps. Parameters are always read-only.
         *
         * Example of reading from a variable:
         *  DateTime startTime = (DateTime) Dts.Variables["System::StartTime"].Value;
         *
         * Example of writing to a variable:
         *  Dts.Variables["User::myStringVariable"].Value = "new value";
         *
         * Example of reading from a package parameter:
         *  int batchId = (int) Dts.Variables["$Package::batchId"].Value;
         * 
         * Example of reading from a project parameter:
         *  int batchId = (int) Dts.Variables["$Project::batchId"].Value;
         *
         * Example of reading from a sensitive project parameter:
         *  int batchId = (int) Dts.Variables["$Project::batchId"].GetSensitiveValue();
         * */

        #endregion

        #region Help:  Firing Integration Services events from a script
        /* This script task can fire events for logging purposes.
         *
         * Example of firing an error event:
         *  Dts.Events.FireError(18, "Process Values", "Bad value", "", 0);
         *
         * Example of firing an information event:
         *  Dts.Events.FireInformation(3, "Process Values", "Processing has started", "", 0, ref fireAgain)
         *
         * Example of firing a warning event:
         *  Dts.Events.FireWarning(14, "Process Values", "No values received for input", "", 0);
         * */
        #endregion

        #region Help:  Using Integration Services connection managers in a script
        /* Some types of connection managers can be used in this script task.  See the topic
         * "Working with Connection Managers Programatically" for details.
         *
         * Example of using an ADO.Net connection manager:
         *  object rawConnection = Dts.Connections["Sales DB"].AcquireConnection(Dts.Transaction);
         *  SqlConnection myADONETConnection = (SqlConnection)rawConnection;
         *  //Use the connection in some code here, then release the connection
         *  Dts.Connections["Sales DB"].ReleaseConnection(rawConnection);
         *
         * Example of using a File connection manager
         *  object rawConnection = Dts.Connections["Prices.zip"].AcquireConnection(Dts.Transaction);
         *  string filePath = (string)rawConnection;
         *  //Use the connection in some code here, then release the connection
         *  Dts.Connections["Prices.zip"].ReleaseConnection(rawConnection);
         * */
        #endregion


             /// <summary>
        /// This method is called when this script task executes in the control flow.
        /// Before returning from this method, set the value of Dts.TaskResult to indicate success or failure.
        /// To open Help, press F1.
        /// </summary>
             public void Main()
             {
            AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(CurrentDomain_AssemblyResolve);
                    // TODO: Add your code here
            GetBucketFiles();
                    Dts.TaskResult = (int)ScriptResults.Success;
             }
        private void GetBucketFiles()
        {
            String ak = “AmazonKey”
            String sk = "AmazonSecretKey";

            string BUCKET_NAME = @"BUCKETNAME";
             AmazonS3Client client = new AmazonS3Client(ak,sk,Amazon.RegionEndpoint.USEast1);

            // List all objects
            ListObjectsRequest listRequest = new ListObjectsRequest
            {
                BucketName = BUCKET_NAME,
                MaxKeys = 1000
              
                
            };

            ListObjectsResponse listResponse;
            do
            {
                // Get a list of objects
                listResponse = client.ListObjects(listRequest);
                foreach (S3Object obj in listResponse.S3Objects)
                {
                    if (obj.Key.EndsWith(".log"))
                    {
                        String FileName = obj.Key;
                        GetObjectRequest request = new GetObjectRequest
                        {
                            BucketName = BUCKET_NAME,
                            Key =  FileName
                        };

                        using (GetObjectResponse response = client.GetObject(request))
                        {
                            string dest = Path.Combine(@"DirectoryPath", FileName.Replace(@"DirectoryKey",""));
                            if (!File.Exists(dest))
                            {
                                response.WriteResponseStreamToFile(dest);
                            }
                          
                        }
                    }
                   
                   
                }

               } while (listResponse.IsTruncated);
        }

        #region ScriptResults declaration
        /// <summary>
        /// This enum provides a convenient shorthand within the scope of this class for setting the
        /// result of the script.
        ///
        /// This code was generated automatically.
        /// </summary>
        enum ScriptResults
        {
            Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
            Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
        };
        #endregion
        static System.Reflection.Assembly CurrentDomain_AssemblyResolve(object sender, ResolveEventArgsargs)
        {

            if (args.Name.Contains("AWSSDK.Core"))
            {
                string path = @"C:\Program Files (x86)\AWS SDK for .NET\bin\Net45\";
                return System.Reflection.Assembly.LoadFile(System.IO.Path.Combine(path,"AWSSDK.Core.dll"));
            }

            /*if (args.Name.Contains("AWSSDK.KeyManagementService"))
            {
                string path = @"C:\Program Files (x86)\AWS SDK for .NET\bin\Net45\";
                return System.Reflection.Assembly.LoadFile(System.IO.Path.Combine(path, "AWSSDK.KeyManagementService.dll"));
            }*/
            if (args.Name.Contains("AWSSDK.S3"))
            {
                string path = @"C:\Program Files (x86)\AWS SDK for .NET\bin\Net45\";
                return System.Reflection.Assembly.LoadFile(System.IO.Path.Combine(path,"AWSSDK.S3.dll"));
            }
            return null;
        }

       }
}

March 14, 2017

Streaming Data Into AWS S3 using KMS Encryption

After a few day of coding, finally I have develop a script component that can upload files into S3 aws buckets, it was a pain to do it this way, if I have a more time, next time it will be done as a Java service instead.
Cheers.





using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Security.Cryptography;
using Amazon.KeyManagementService;
using Amazon.KeyManagementService.Model;
using Amazon.S3;
using Amazon.S3.Encryption;
using Amazon.S3.Model;

namespace ConsoleEncryptStream
{
    class Program
    {
        static void Main(string[] args)
        {
            String ak = Amazonkey
            String sk = amazonSecretKey

          String arn = KMS_ALIAS;
          Stream input = new FileStream(@"C:\temp\test.txt", FileMode.Open, FileAccess.Read);
          Stream output = new FileStream(@"\\test.kms", FileMode.Create, FileAccess.Write);
        //  FileUpload(ak, sk, arn);
        //  ClientEncryption(input,ak, sk, arn);
         clientDecryption(ak, sk, arn);
         // Encrypt(input, output, ak, sk, arn);
         // Decrypt(input,output,ak,sk,arn);
        }

        static void FileUpload(string AmazonKey, string AmazonSecretKey, string arn)
        {
            try
            {

                var client = new AmazonS3Client(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1);
                string BUCKET_NAME = "BucketName";
           
               /* ListBucketsResponse response = client.ListBuckets();
                bool found = false;
                foreach (S3Bucket bucket in response.Buckets)
                {
                    if (bucket.BucketName == BUCKET_NAME)
                    {
                        found = true;
                        break;
                    }
                }
                if (found == false)
                {
                    client.PutBucket(new PutBucketRequest().BucketName = BUCKET_NAME);
                }*/
                String S3_KEY = "create/FileList.txt";
                PutObjectRequest request =( new PutObjectRequest(){
                    BucketName = BUCKET_NAME,
                  Key = S3_KEY,
                ContentBody = "This is body of S3 object.",
            });
                client.PutObject(request);
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message.ToString());
                Console.Read();
            }
        }
        static void ClientEncryption(Stream input, string AmazonKey, string AmazonSecretKey, string arn)
        {
            try
            {
                string BUCKET_NAME = "BucketName";
                using (var client = new AmazonKeyManagementServiceClient(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1))
                using (var algorithm = new KMSAlgorithm(client, arn))
                {
                    var materials = new EncryptionMaterials(algorithm);
                    var s3client = new AmazonS3EncryptionClient(AmazonKey,AmazonSecretKey,Amazon.RegionEndpoint.USEast1,materials);
  
                   PutObjectRequest request =( new PutObjectRequest()
                    {
                        BucketName = BUCKET_NAME,
                        Key = "create/outfile.txt",
                        InputStream = input,
                       // CannedACL = S3CannedACL.PublicRead,
                       
                    });
                   s3client.PutObject(request);
                   
                }
                Console.WriteLine("File Uploaded...");
                Console.Read();
                      
            }
           
            catch (Exception e)
            {
                Console.WriteLine(e.Message.ToString());
                Console.Read();
            }
           
        }
        static void clientDecryption (string AmazonKey, string AmazonSecretKey, string arn)
        {
            try
            {
               
                string BUCKET_NAME = "BucketName";
                string FileName = "create/test.txt";
                using (var client = new AmazonKeyManagementServiceClient(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1))
                using (var algorithm = new KMSAlgorithm(client, arn))
                {
                    var materials = new EncryptionMaterials(algorithm);
                    var s3client = new AmazonS3EncryptionClient(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1, materials);

                    GetObjectRequest request = (new GetObjectRequest()
                    {
                        BucketName = BUCKET_NAME,
                        Key = FileName,
                       
                        // CannedACL = S3CannedACL.PublicRead,

                    });
                    using (var response = s3client.GetObject(request))
                    {
                        string dest = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), FileName);
                        if (!File.Exists(dest))
                        {
                            response.WriteResponseStreamToFile(dest);
                        }
                    }
                   // s3client.PutObject(request);

                }
                Console.WriteLine("File downloaded...");
                Console.Read();

            }

            catch (Exception e)
            {
                Console.WriteLine(e.Message.ToString());
                Console.Read();
            }
        }
        static void Encrypt(Stream input, Stream output, string AmazonKey, string AmazonSecretKey, string arn)
        {
            try
            {
                using (var kms = new AmazonKeyManagementServiceClient(AmazonKey, AmazonSecretKey, Amazon.RegionEndpoint.USEast1))
                using (var algorithm = Aes.Create())
                {
                    var dataKey = kms.GenerateDataKey(new GenerateDataKeyRequest
                    {
                        KeyId = arn,
                        KeySpec = DataKeySpec.AES_256
                    });
                    output.WriteByte((byte)dataKey.CiphertextBlob.Length);

Contact Form

Name

Email *

Message *