"I hear and I forget. I see and I remember. I do and I understand."
Confucius
Jorge Novo Development Solutions,Ideas, and Crazy Thoughts.
Email: ETLDEVDBA(at)gmail.com
Tweeter: @ETLDEVDBA
October 16, 2017
Postgresql Insert into Multiple Tables using void.
When you think you know everything here comes something simple to keep your hands dirty, a developer try to translate a ms sql procedure into a postgresql function and eventually it was pass to me, so I wrote something similar to below with a special caveat.
Enjoy....
CREATE OR REPLACE FUNCTION SchemaName.FunctionName_Save(
_parameter1 character varying DEFAULT NULL::character varying,
_parameter2 character varying DEFAULT NULL::character varying,
_parameter3 timestamp without time zone DEFAULT NULL::timestamp without time zone,
__parameter4 timestamp without time zone DEFAULT NULL::timestamp without time zone,
_parameter5 character varying DEFAULT NULL::character varying,
_parameter6 character varying DEFAULT NULL::character varying,
_parameter7 character varying DEFAULT NULL::character varying,
_parameter8 character varying DEFAULT NULL::character varying,
_parameter9 character varying DEFAULT NULL::character varying,
_parameter0 character varying DEFAULT NULL::character varying,
_time timestamp without time zone DEFAULT NULL::timestamp without time zone,
_result character varying DEFAULT NULL::character varying,
_details character varying DEFAULT NULL::character varying,
_queue character varying DEFAULT NULL::character varying)
RETURNS void
LANGUAGE 'plpgsql'
COST 100
VOLATILE
AS $BODY$
declare
_local01 int;
_local02 int;
_local03 int;
_local04 int;
_local05 bigint;
_local06 bigint;
_local07 bigint;
begin
if not exists ( select 1 from SchemaName.TableName0 where lower (name) = lower(_parameter1) )
THEN
insert into SchemaName.TableName0 (name,insertdate)
values (_parameter1,__parameter4)
returning SchemaName.TableName0.TableName0id into _local01;
else
select SchemaName.TableName0.TableName0id into _local01 from SchemaName.TableName0 where lower (ColumnName) = lower(_parameter1);
end if;
if not exists ( select 1 from SchemaName.TableName01 where lower (ColumnName) = lower(_parameter2) )
THEN
insert into SchemaName.TableName01(name)
values (_parameter2)
returning SchemaName.TableName01.TableName01id into _local02;
else
select SchemaName.TableName01.TableName01id into _local02 from SchemaName.TableName01 where lower (name) = lower(_parameter2);
end if;
if not exists ( select 1 from SchemaName..TableName02 where lower (name) = lower(_parameter5) )
THEN
insert into SchemaName..TableName02(name)
values (_parameter5)
returning SchemaName..TableName02..TableName02id into _local03;
ELSE
select SchemaName..TableName02..TableName02id into _local03 from SchemaName..TableName02 where lower (ColumnName) = lower(_parameter5);
end if;
if not exists( select 1 from SchemaName.queue where lower (queueaddress) = lower(_queue))
then
insert into SchemaName.queue(column,column)
values (_queue,__parameter4)
returning SchemaName.queue.queueid into _local04;
else
select SchemaName.queue.queueid into _local04 from SchemaName.queue where lower (ColumnName) = lower(_queue);
end if;
_local05 := nextval('SchemaName.TableName04_seq'::regclass);
INSERT INTO SchemaName.TableName04(
Columns,columns,columns)
values
( _local02,_parameter6,_parameter3e,__parameter4,_local04);
--- returning SchemaName.TableName04.TableName04id into _local05;
_local06 := nextval('SchemaName.TableName04_local06_seq'::regclass);
INSERT INTO SchemaName.TableName04details(columns,columns,columns)
values( _local05,_parameter7,_parameter8,__parameter4)
returning SchemaName.TableName04details.TableName04detailsid into _local06;
INSERT INTO SchemaName..TableName06
( columns,columns,columns,columns,...)
VALUES
( _local01, _parameter0, _parameter9, _local05, _Time, _local03, _Result, _Details);
end;
$BODY$;
ALTER FUNCTION SchemaName.FunctionName_Save(character varying, character varying, timestamp without time zone, timestamp without time zone, character varying, character varying, character varying, character varying, character varying, character varying, timestamp without time zone, character varying, character varying, character varying)
OWNER TO db_owner;
GRANT EXECUTE ON FUNCTION SchemaName.FunctionName_Save(character varying, character varying, timestamp without time zone, timestamp without time zone, character varying, character varying, character varying, character varying, character varying, character varying, timestamp without time zone, character varying, character varying, character varying) TO db_datawriter;
GRANT EXECUTE ON FUNCTION SchemaName.FunctionName_Save(character varying, character varying, timestamp without time zone, timestamp without time zone, character varying, character varying, character varying, character varying, character varying, character varying, timestamp without time zone, character varying, character varying, character varying) TO PUBLIC;
GRANT EXECUTE ON FUNCTION SchemaName.FunctionName_Save(character varying, character varying, timestamp without time zone, timestamp without time zone, character varying, character varying, character varying, character varying, character varying, character varying, timestamp without time zone, character varying, character varying, character varying) TO db_executor;
GRANT EXECUTE ON FUNCTION SchemaName.FunctionName_Save(character varying, character varying, timestamp without time zone, timestamp without time zone, character varying, character varying, character varying, character varying, character varying, character varying, timestamp without time zone, character varying, character varying, character varying) TO db_owner;
August 14, 2017
SSIS Json Serializer for SQL Server 2014 and below
#region Help: Introduction to the Script Component
/* The Script Component allows you to perform virtually any operation that can be accomplished in
* a .Net application within the context of an Integration Services data flow.
*
* Expand the other regions which have "Help" prefixes for examples of specific ways to use
* Integration Services features within this script component. */
#endregion
#region Namespaces
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Pipeline.Wrapper;
using Microsoft.SqlServer.Dts.Runtime.Wrapper;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using System.IO;
using System.Text;
using System.Xml;
#endregion
/// <summary>
/// This is the class to which to add your code. Do not change the name, attributes, or parent
/// of this class.
/// </summary>
[Microsoft.SqlServer.Dts.Pipeline.SSISScriptComponentEntryPointAttribute]
public class ScriptMain : UserComponent
{
#region Help: Using Integration Services variables and parameters
/* To use a variable in this script, first ensure that the variable has been added to
* either the list contained in the ReadOnlyVariables property or the list contained in
* the ReadWriteVariables property of this script component, according to whether or not your
* code needs to write into the variable. To do so, save this script, close this instance of
* Visual Studio, and update the ReadOnlyVariables and ReadWriteVariables properties in the
* Script Transformation Editor window.
* To use a parameter in this script, follow the same steps. Parameters are always read-only.
*
* Example of reading from a variable or parameter:
* DateTime startTime = Variables.MyStartTime;
*
* Example of writing to a variable:
* Variables.myStringVariable = "new value";
*/
#endregion
#region Help: Using Integration Services Connnection Managers
/* Some types of connection managers can be used in this script component. See the help topic
* "Working with Connection Managers Programatically" for details.
*
* To use a connection manager in this script, first ensure that the connection manager has
* been added to either the list of connection managers on the Connection Managers page of the
* script component editor. To add the connection manager, save this script, close this instance of
* Visual Studio, and add the Connection Manager to the list.
*
* If the component needs to hold a connection open while processing rows, override the
* AcquireConnections and ReleaseConnections methods.
*
* Example of using an ADO.Net connection manager to acquire a SqlConnection:
* object rawConnection = Connections.SalesDB.AcquireConnection(transaction);
* SqlConnection salesDBConn = (SqlConnection)rawConnection;
*
* Example of using a File connection manager to acquire a file path:
* object rawConnection = Connections.Prices_zip.AcquireConnection(transaction);
* string filePath = (string)rawConnection;
*
* Example of releasing a connection manager:
* Connections.SalesDB.ReleaseConnection(rawConnection);
*/
#endregion
#region Help: Firing Integration Services Events
/* This script component can fire events.
*
* Example of firing an error event:
* ComponentMetaData.FireError(10, "Process Values", "Bad value", "", 0, out cancel);
*
* Example of firing an information event:
* ComponentMetaData.FireInformation(10, "Process Values", "Processing has started", "", 0, fireAgain);
*
* Example of firing a warning event:
* ComponentMetaData.FireWarning(10, "Process Values", "No rows were received", "", 0);
*/
#endregion
/// <summary>
/// This method is called once, before rows begin to be processed in the data flow.
///
/// You can remove this method if you don't need to do anything here.
/// </summary>
public override void PreExecute()
{
base.PreExecute();
/*
* Add your code here
*/
}
/// <summary>
/// This method is called after all the rows have passed through this component.
///
/// You can delete this method if you don't need to do anything here.
/// </summary>
public override void PostExecute()
{
base.PostExecute();
/*
* Add your code here
*/
}
/// <summary>
/// This method is called once for every row that passes through the component from Input0.
///
/// Example of reading a value from a column in the the row:
/// string zipCode = Row.ZipCode
///
/// Example of writing a value to a column in the row:
/// Row.ZipCode = zipCode
/// </summary>
/// <param name="Row">The row that is currently passing through the component</param>
public override void Input0_ProcessInputRow(Input0Buffer Row)
{
try
{
var dlenght = Convert.ToInt32(Row.XmlData.Length);// Get Length side of xml record
var ddata = Row.XmlData.GetBlobData(0, dlenght); // Get Data as a block of bytes
var xdata = Encoding.UTF8.GetString(ddata); // Get String Encoded to UFT8
StringBuilder Rawjson = new StringBuilder(); // Get String Builder
XmlDataDocument doc = new XmlDataDocument(); // Create Xml document
doc.LoadXml(xdata); // Load Document
XmlElement root = doc.DocumentElement;
/*
* Block to add missong nodes to the code or remove childrens
* var xNode = doc.SelectSingleNode("Customer/Password");
xNode.ParentNode.RemoveChild(xNode);
var password = doc.CreateElement("Password");
var email = doc.SelectSingleNode("Customer/Email");
root.InsertBefore(password, email);
*
*/
// this is where the magic begin
var settings = new JsonSerializerSettings();
settings.NullValueHandling = NullValueHandling.Include;
byte[] jbyte = ASCIIEncoding.UTF8.GetBytes(JsonConvert.SerializeXmlNode(doc, Newtonsoft.Json.Formatting.None));
Rawjson.Append(Row.CustomerId.ToString() + "|");
Rawjson.Append(Encoding.UTF8.GetString(jbyte));
jbyte = null;
// json.Replace(Password, DBNull.Value.ToString());
byte[] jsonbyte = ASCIIEncoding.UTF8.GetBytes(Rawjson.ToString());
Output0Buffer.AddRow();
Output0Buffer.CustomerId = (int)Row.CustomerId;
Output0Buffer.Json.AddBlobData(jsonbyte);
}
catch (Exception ex)
{
String ErrorMesage = "Error_JsonConversation CustomerId:" + Row.CustomerId.ToString();
bool fireError = true;
IDTSComponentMetaData100 myMetaData;
myMetaData = this.ComponentMetaData;
myMetaData.FireError(0, ErrorMesage, ex.ToString(), string.Empty, 0, out fireError);
}
}
public override void CreateNewOutputRows()
{
/*
Add rows by calling the AddRow method on the member variable named "<Output Name>Buffer".
For example, call MyOutputBuffer.AddRow() if your output was named "MyOutput".
*/
}
}
/* The Script Component allows you to perform virtually any operation that can be accomplished in
* a .Net application within the context of an Integration Services data flow.
*
* Expand the other regions which have "Help" prefixes for examples of specific ways to use
* Integration Services features within this script component. */
#endregion
#region Namespaces
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Pipeline.Wrapper;
using Microsoft.SqlServer.Dts.Runtime.Wrapper;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using System.IO;
using System.Text;
using System.Xml;
#endregion
/// <summary>
/// This is the class to which to add your code. Do not change the name, attributes, or parent
/// of this class.
/// </summary>
[Microsoft.SqlServer.Dts.Pipeline.SSISScriptComponentEntryPointAttribute]
public class ScriptMain : UserComponent
{
#region Help: Using Integration Services variables and parameters
/* To use a variable in this script, first ensure that the variable has been added to
* either the list contained in the ReadOnlyVariables property or the list contained in
* the ReadWriteVariables property of this script component, according to whether or not your
* code needs to write into the variable. To do so, save this script, close this instance of
* Visual Studio, and update the ReadOnlyVariables and ReadWriteVariables properties in the
* Script Transformation Editor window.
* To use a parameter in this script, follow the same steps. Parameters are always read-only.
*
* Example of reading from a variable or parameter:
* DateTime startTime = Variables.MyStartTime;
*
* Example of writing to a variable:
* Variables.myStringVariable = "new value";
*/
#endregion
#region Help: Using Integration Services Connnection Managers
/* Some types of connection managers can be used in this script component. See the help topic
* "Working with Connection Managers Programatically" for details.
*
* To use a connection manager in this script, first ensure that the connection manager has
* been added to either the list of connection managers on the Connection Managers page of the
* script component editor. To add the connection manager, save this script, close this instance of
* Visual Studio, and add the Connection Manager to the list.
*
* If the component needs to hold a connection open while processing rows, override the
* AcquireConnections and ReleaseConnections methods.
*
* Example of using an ADO.Net connection manager to acquire a SqlConnection:
* object rawConnection = Connections.SalesDB.AcquireConnection(transaction);
* SqlConnection salesDBConn = (SqlConnection)rawConnection;
*
* Example of using a File connection manager to acquire a file path:
* object rawConnection = Connections.Prices_zip.AcquireConnection(transaction);
* string filePath = (string)rawConnection;
*
* Example of releasing a connection manager:
* Connections.SalesDB.ReleaseConnection(rawConnection);
*/
#endregion
#region Help: Firing Integration Services Events
/* This script component can fire events.
*
* Example of firing an error event:
* ComponentMetaData.FireError(10, "Process Values", "Bad value", "", 0, out cancel);
*
* Example of firing an information event:
* ComponentMetaData.FireInformation(10, "Process Values", "Processing has started", "", 0, fireAgain);
*
* Example of firing a warning event:
* ComponentMetaData.FireWarning(10, "Process Values", "No rows were received", "", 0);
*/
#endregion
/// <summary>
/// This method is called once, before rows begin to be processed in the data flow.
///
/// You can remove this method if you don't need to do anything here.
/// </summary>
public override void PreExecute()
{
base.PreExecute();
/*
* Add your code here
*/
}
/// <summary>
/// This method is called after all the rows have passed through this component.
///
/// You can delete this method if you don't need to do anything here.
/// </summary>
public override void PostExecute()
{
base.PostExecute();
/*
* Add your code here
*/
}
/// <summary>
/// This method is called once for every row that passes through the component from Input0.
///
/// Example of reading a value from a column in the the row:
/// string zipCode = Row.ZipCode
///
/// Example of writing a value to a column in the row:
/// Row.ZipCode = zipCode
/// </summary>
/// <param name="Row">The row that is currently passing through the component</param>
public override void Input0_ProcessInputRow(Input0Buffer Row)
{
try
{
var dlenght = Convert.ToInt32(Row.XmlData.Length);// Get Length side of xml record
var ddata = Row.XmlData.GetBlobData(0, dlenght); // Get Data as a block of bytes
var xdata = Encoding.UTF8.GetString(ddata); // Get String Encoded to UFT8
StringBuilder Rawjson = new StringBuilder(); // Get String Builder
XmlDataDocument doc = new XmlDataDocument(); // Create Xml document
doc.LoadXml(xdata); // Load Document
XmlElement root = doc.DocumentElement;
/*
* Block to add missong nodes to the code or remove childrens
* var xNode = doc.SelectSingleNode("Customer/Password");
xNode.ParentNode.RemoveChild(xNode);
var password = doc.CreateElement("Password");
var email = doc.SelectSingleNode("Customer/Email");
root.InsertBefore(password, email);
*
*/
// this is where the magic begin
var settings = new JsonSerializerSettings();
settings.NullValueHandling = NullValueHandling.Include;
byte[] jbyte = ASCIIEncoding.UTF8.GetBytes(JsonConvert.SerializeXmlNode(doc, Newtonsoft.Json.Formatting.None));
Rawjson.Append(Row.CustomerId.ToString() + "|");
Rawjson.Append(Encoding.UTF8.GetString(jbyte));
jbyte = null;
// json.Replace(Password, DBNull.Value.ToString());
byte[] jsonbyte = ASCIIEncoding.UTF8.GetBytes(Rawjson.ToString());
Output0Buffer.AddRow();
Output0Buffer.CustomerId = (int)Row.CustomerId;
Output0Buffer.Json.AddBlobData(jsonbyte);
}
catch (Exception ex)
{
String ErrorMesage = "Error_JsonConversation CustomerId:" + Row.CustomerId.ToString();
bool fireError = true;
IDTSComponentMetaData100 myMetaData;
myMetaData = this.ComponentMetaData;
myMetaData.FireError(0, ErrorMesage, ex.ToString(), string.Empty, 0, out fireError);
}
}
public override void CreateNewOutputRows()
{
/*
Add rows by calling the AddRow method on the member variable named "<Output Name>Buffer".
For example, call MyOutputBuffer.AddRow() if your output was named "MyOutput".
*/
}
}
June 28, 2017
PostgreSQL Quarterly Partition Function
Here is some quick code on how to create a partition function using inherit tables on PostgreSql, is simple but hopefully this will save some poor soul the documentation reading and for me save me some time in the future, currently things been very busy working in Ms Sql and PostgreSql doing several conversions from one DB to the other.
1a.-Create sequence
1b.- Create table
2.-Create function
3.-create trigger
-- FUNCTION:
TableSchemaNameHere.TableName_partition_insert()
-- DROP FUNCTION TableSchemaNameHere.event_partiont_insert();
CREATE SEQUENCE seq_TableSchemaNameHere.TableNameHere_TableColumn
minvalue 1 nomaxvalue increment by 1;
create table TableSchemaNameHere.Tablename
(
PKID int DEFAULT nextval('seq_TableSchemaNameHere.TableNameHere_TableColumn') NOT NULL,
insertdate timestamp(6) without time zone default now(),
time timestamp without time zone,
constraint pk_ primary key (pkid)
)
CREATE or Replace FUNCTION TableSchemaNameHere.TableName_partiont_insert()
RETURNS trigger
LANGUAGE 'plpgsql'
COST 100.0
VOLATILE NOT LEAKPROOF
AS $BODY$
declare
declare
_partition_date varchar(50);
_partition varchar(80);
_IsNewChild boolean;
_current_day int;
_first_day varchar(2) := '01';
_last_day varchar(2);
_pick_day varchar(2);
_tablename varchar(80);
begin
-- get current day
_current_day := date_part('day',new.time);
-- find last day of the month function
_last_day := date_part ('day',TableSchemaNameHere.last_day (new.time::date));
-- get YYYY_MM format
_partition_date := to_char (NEW.time,'YYYY_MM');
-- find if first or second sequence
if _current_day < 16
then
_pick_day := _first_day;
_partition_date := _partition_date||'_'|| _first_day;
else
_pick_day = _last_day;
_partition_date := _partition_date||'_'|| _last_day;
end if;
_partition := TG_TABLE_NAME || '_' || _partition_date;
_tablename := _partition;
if not exists ( select 1 from pg_tables where schemaname= TG_TABLE_SCHEMA and tablename= _partition)
then
RAISE NOTICE 'A partition has been created %',TG_TABLE_SCHEMA ||'.'|| _partition;
_IsNewChild = TRUE;
EXECUTE 'create table '|| TG_TABLE_SCHEMA ||'.'|| _partition || ' (check( to_char (time,''YYYY_MM'||'_'|| _pick_day||''') = '''|| to_char (NEW.time,'YYYY_MM')||'_'||_pick_day||''')) INHERITS ( TableSchemaNameHere.TableNamehere);';
end if;
execute 'insert into '|| TG_TABLE_SCHEMA || '.' || _partition || ' SELECT(' || TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').* RETURNING eventid;';
if (_IsNewChild)
then
-- Add primary key
EXECUTE format('ALTER TABLE '||TG_TABLE_SCHEMA||'.%s ADD PRIMARY KEY(pkid)', _partition);
-- Assign owner of of inherited table
EXECUTE format('ALTER TABLE '||TG_TABLE_SCHEMA||'.%s OWNER TO db_owner', _partition);
-- Add FK promo_batch_id
EXECUTE format('ALTER TABLE '||TG_TABLE_SCHEMA||'.%s ADD CONSTRAINT FK_%s FOREIGN KEY(columnId) REFERENCES TableSchemaNameHere.fkid (fkid) MATCH FULL',_tablename, _tablename);
-- Need to define indexes for inherited tables
EXECUTE format('CREATE INDEX ix_%s_EventTypeID ON '||TG_TABLE_SCHEMA||'.%s USING btree ("indexkeycolumn")', _tablename, _tablename);
end if;
RETURN NULL;
end
$BODY$;
----Create trigger
CREATE TRIGGER tr_TableName_Partition
BEFORE INSERT ON TableSchemaNameHere.TableNameHere
FOR EACH ROW EXECUTE PROCEDURE myschema.server_partition_function();
1a.-Create sequence
1b.- Create table
2.-Create function
3.-create trigger
-- FUNCTION:
TableSchemaNameHere.TableName_partition_insert()
-- DROP FUNCTION TableSchemaNameHere.event_partiont_insert();
CREATE SEQUENCE seq_TableSchemaNameHere.TableNameHere_TableColumn
minvalue 1 nomaxvalue increment by 1;
create table TableSchemaNameHere.Tablename
(
PKID int DEFAULT nextval('seq_TableSchemaNameHere.TableNameHere_TableColumn') NOT NULL,
insertdate timestamp(6) without time zone default now(),
time timestamp without time zone,
constraint pk_ primary key (pkid)
)
CREATE or Replace FUNCTION TableSchemaNameHere.TableName_partiont_insert()
RETURNS trigger
LANGUAGE 'plpgsql'
COST 100.0
VOLATILE NOT LEAKPROOF
AS $BODY$
declare
declare
_partition_date varchar(50);
_partition varchar(80);
_IsNewChild boolean;
_current_day int;
_first_day varchar(2) := '01';
_last_day varchar(2);
_pick_day varchar(2);
_tablename varchar(80);
begin
-- get current day
_current_day := date_part('day',new.time);
-- find last day of the month function
_last_day := date_part ('day',TableSchemaNameHere.last_day (new.time::date));
-- get YYYY_MM format
_partition_date := to_char (NEW.time,'YYYY_MM');
-- find if first or second sequence
if _current_day < 16
then
_pick_day := _first_day;
_partition_date := _partition_date||'_'|| _first_day;
else
_pick_day = _last_day;
_partition_date := _partition_date||'_'|| _last_day;
end if;
_partition := TG_TABLE_NAME || '_' || _partition_date;
_tablename := _partition;
if not exists ( select 1 from pg_tables where schemaname= TG_TABLE_SCHEMA and tablename= _partition)
then
RAISE NOTICE 'A partition has been created %',TG_TABLE_SCHEMA ||'.'|| _partition;
_IsNewChild = TRUE;
EXECUTE 'create table '|| TG_TABLE_SCHEMA ||'.'|| _partition || ' (check( to_char (time,''YYYY_MM'||'_'|| _pick_day||''') = '''|| to_char (NEW.time,'YYYY_MM')||'_'||_pick_day||''')) INHERITS ( TableSchemaNameHere.TableNamehere);';
end if;
execute 'insert into '|| TG_TABLE_SCHEMA || '.' || _partition || ' SELECT(' || TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME || ' ' || quote_literal(NEW) || ').* RETURNING eventid;';
if (_IsNewChild)
then
-- Add primary key
EXECUTE format('ALTER TABLE '||TG_TABLE_SCHEMA||'.%s ADD PRIMARY KEY(pkid)', _partition);
-- Assign owner of of inherited table
EXECUTE format('ALTER TABLE '||TG_TABLE_SCHEMA||'.%s OWNER TO db_owner', _partition);
-- Add FK promo_batch_id
EXECUTE format('ALTER TABLE '||TG_TABLE_SCHEMA||'.%s ADD CONSTRAINT FK_%s FOREIGN KEY(columnId) REFERENCES TableSchemaNameHere.fkid (fkid) MATCH FULL',_tablename, _tablename);
-- Need to define indexes for inherited tables
EXECUTE format('CREATE INDEX ix_%s_EventTypeID ON '||TG_TABLE_SCHEMA||'.%s USING btree ("indexkeycolumn")', _tablename, _tablename);
end if;
RETURN NULL;
end
$BODY$;
----Create trigger
CREATE TRIGGER tr_TableName_Partition
BEFORE INSERT ON TableSchemaNameHere.TableNameHere
FOR EACH ROW EXECUTE PROCEDURE myschema.server_partition_function();
May 31, 2017
Get objects from AWS S3
Continue to work on pulling and pushing objects into S3 aws so here is a quick and dirty code cannibalize around...
#region Help: Introduction to the script task
/* The Script Task allows you to perform virtually any operation that can be accomplished in
* a .Net application within the context of an Integration Services control flow.
*
* Expand the other regions which have "Help" prefixes for examples of specific ways to use
* Integration Services features within this script task. */
#endregion
#region Namespaces
using System;
using System.Data;
using Microsoft.SqlServer.Dts. Runtime;
using System.Windows.Forms;
using System.IO;
using Amazon.S3;
using Amazon.S3.Model;
using Amazon;
#endregion
namespace ST_ 951ab847fc194587b642a51853cb75 36
{
/// <summary>
/// ScriptMain is the entry point class of the script. Do not change the name, attributes,
/// or parent of this class.
/// </summary>
[Microsoft.SqlServer.Dts. Tasks.ScriptTask.SSISScriptTas kEntryPointAttribute]
public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks. ScriptTask.VSTARTScriptObjectM odelBase
{
#region Help: Using Integration Services variables and parameters in a script
/* To use a variable in this script, first ensure that the variable has been added to
* either the list contained in the ReadOnlyVariables property or the list contained in
* the ReadWriteVariables property of this script task, according to whether or not your
* code needs to write to the variable. To add the variable, save this script, close this instance of
* Visual Studio, and update the ReadOnlyVariables and
* ReadWriteVariables properties in the Script Transformation Editor window.
* To use a parameter in this script, follow the same steps. Parameters are always read-only.
*
* Example of reading from a variable:
* DateTime startTime = (DateTime) Dts.Variables["System:: StartTime"].Value;
*
* Example of writing to a variable:
* Dts.Variables["User:: myStringVariable"].Value = "new value";
*
* Example of reading from a package parameter:
* int batchId = (int) Dts.Variables["$Package:: batchId"].Value;
*
* Example of reading from a project parameter:
* int batchId = (int) Dts.Variables["$Project:: batchId"].Value;
*
* Example of reading from a sensitive project parameter:
* int batchId = (int) Dts.Variables["$Project:: batchId"].GetSensitiveValue();
* */
#endregion
#region Help: Firing Integration Services events from a script
/* This script task can fire events for logging purposes.
*
* Example of firing an error event:
* Dts.Events.FireError(18, "Process Values", "Bad value", "", 0);
*
* Example of firing an information event:
* Dts.Events.FireInformation(3, "Process Values", "Processing has started", "", 0, ref fireAgain)
*
* Example of firing a warning event:
* Dts.Events.FireWarning(14, "Process Values", "No values received for input", "", 0);
* */
#endregion
#region Help: Using Integration Services connection managers in a script
/* Some types of connection managers can be used in this script task. See the topic
* "Working with Connection Managers Programatically" for details.
*
* Example of using an ADO.Net connection manager:
* object rawConnection = Dts.Connections["Sales DB"].AcquireConnection(Dts. Transaction);
* SqlConnection myADONETConnection = (SqlConnection)rawConnection;
* //Use the connection in some code here, then release the connection
* Dts.Connections["Sales DB"].ReleaseConnection( rawConnection);
*
* Example of using a File connection manager
* object rawConnection = Dts.Connections["Prices.zip"]. AcquireConnection(Dts. Transaction);
* string filePath = (string)rawConnection;
* //Use the connection in some code here, then release the connection
* Dts.Connections["Prices.zip"]. ReleaseConnection( rawConnection);
* */
#endregion
/// <summary>
/// This method is called when this script task executes in the control flow.
/// Before returning from this method, set the value of Dts.TaskResult to indicate success or failure.
/// To open Help, press F1.
/// </summary>
public void Main()
{
AppDomain.CurrentDomain. AssemblyResolve += new ResolveEventHandler( CurrentDomain_AssemblyResolve) ;
// TODO: Add your code here
GetBucketFiles();
Dts.TaskResult = (int)ScriptResults.Success;
}
private void GetBucketFiles()
{
String ak = “AmazonKey”
String sk = "AmazonSecretKey";
string BUCKET_NAME = @"BUCKETNAME";
AmazonS3Client client = new AmazonS3Client(ak,sk,Amazon.Re gionEndpoint.USEast1);
// List all objects
ListObjectsRequest listRequest = new ListObjectsRequest
{
BucketName = BUCKET_NAME,
MaxKeys = 1000
};
ListObjectsResponse listResponse;
do
{
// Get a list of objects
listResponse = client.ListObjects( listRequest);
foreach (S3Object obj in listResponse.S3Objects)
{
if (obj.Key.EndsWith(".log"))
{
String FileName = obj.Key;
GetObjectRequest request = new GetObjectRequest
{
BucketName = BUCKET_NAME,
Key = FileName
};
using (GetObjectResponse response = client.GetObject(request))
{
string dest = Path.Combine(@"DirectoryPath", FileName.Replace(@"DirectoryKey","" ));
if (!File.Exists(dest))
{
}
}
}
}
} while (listResponse.IsTruncated);
}
#region ScriptResults declaration
/// <summary>
/// This enum provides a convenient shorthand within the scope of this class for setting the
/// result of the script.
///
/// This code was generated automatically.
/// </summary>
enum ScriptResults
{
Success = Microsoft.SqlServer.Dts. Runtime.DTSExecResult.Success,
Failure = Microsoft.SqlServer.Dts. Runtime.DTSExecResult.Failure
};
#endregion
static System.Reflection.Assembly CurrentDomain_AssemblyResolve( object sender, ResolveEventArgsargs)
{
if (args.Name.Contains("AWSSDK. Core"))
{
string path = @"C:\Program Files (x86)\AWS SDK for .NET\bin\Net45\";
return System.Reflection.Assembly. LoadFile(System.IO.Path. Combine(path,"AWSSDK.Core.dll"));
}
/*if (args.Name.Contains("AWSSDK. KeyManagementService"))
{
string path = @"C:\Program Files (x86)\AWS SDK for .NET\bin\Net45\";
return System.Reflection.Assembly. LoadFile(System.IO.Path. Combine(path, "AWSSDK.KeyManagementService. dll"));
}*/
if (args.Name.Contains("AWSSDK. S3"))
{
string path = @"C:\Program Files (x86)\AWS SDK for .NET\bin\Net45\";
return System.Reflection.Assembly. LoadFile(System.IO.Path. Combine(path,"AWSSDK.S3.dll"));
}
return null;
}
}
}
Subscribe to:
Posts (Atom)