Skip to content

Task/rdmp 376 extraction structural changes #2343

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
JFriel wants to merge 15 commits into
base: develop
Choose a base branch
from
Draft

Task/rdmp 376 extraction structural changes #2343

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
acfae60
working add
JFriel Jun 3, 2026
ba4865d
col working
JFriel Jun 4, 2026
0a6e3f3
null columns
JFriel Jun 4, 2026
c24f37d
working triggr
JFriel Jun 4, 2026
2cfbe07
tidy up
JFriel Jun 4, 2026
c0c381e
update codeql
JFriel Jun 9, 2026
6181033
add tests
JFriel Jun 10, 2026
f7cb8b8
update test pipelines
JFriel Jun 10, 2026
e733b9c
Merge branch 'develop' into task/RDMP-376-extraction-structural-changes
JFriel Jul 2, 2026
21bcd74
update merge component
JFriel Jul 2, 2026
4b7bd9f
add tests
JFriel Jul 2, 2026
61aceed
update merge
JFriel Jul 6, 2026
8ddd426
fix some tests
JFriel Jul 6, 2026
be04f3f
update tests
JFriel Jul 6, 2026
97eeb0c
update test
JFriel Jul 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 70 additions & 15 deletions ...e/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

using Amazon.Auth.AccessControlPolicy;
using FAnsi.Discovery;
using NPOI.SS.Formula.Functions;
using Rdmp.Core.CommandExecution;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataExport.Data;
Expand All @@ -16,6 +17,7 @@
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.Job.Scheduling;
using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations;
using Rdmp.Core.DataLoad.Triggers;
using Rdmp.Core.DataLoad.Triggers.Exceptions;
using Rdmp.Core.DataLoad.Triggers.Implementations;
using Rdmp.Core.MapsDirectlyToDatabaseTable;
Expand Down Expand Up @@ -169,6 +171,14 @@
LinesWritten += toProcess.Rows.Count;
}


private bool hasStructuralChanges(DataTable source, DiscoveredTable destination)
{
var sourceColumns = source.Columns.Cast<DataColumn>().Select(c => c.ColumnName).ToList();
var destinationColumns = destination.DiscoverColumns().Select(c => c.GetRuntimeName()).ToList();
return !sourceColumns.All(destinationColumns.Contains) || !destinationColumns.All(sourceColumns.Contains);
}

private DataTableUploadDestination PrepareDestination(IDataLoadEventListener listener, DataTable toProcess)
{
//see if the user has entered an extraction server/database
Expand All @@ -191,7 +201,17 @@
if (existing.Exists())
{
var hasPKs = existing.DiscoverColumns().Any(col => col.IsPrimaryKey);

TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer);
var implementor = triggerFactory.Create(existing);
bool present;
try
{
present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled;
}
catch (TriggerMissingException)
{
present = false;
}
if (!AlwaysDropExtractionTables)
{
//check the PKs are the same
Expand All @@ -205,7 +225,51 @@
Source PKs: {string.Join(", ", rdmpPKs)}
Destination PKs: {string.Join(", ", remotePKs)}
"""));
return null;
return null;//todo this error could be better
}
if (hasStructuralChanges(toProcess, existing))
{
var sourceColumns = toProcess.Columns.Cast<DataColumn>().Select(c => c.ColumnName).ToList();
var destinationColumns = existing.DiscoverColumns().Select(c => c.GetRuntimeName()).ToList();

//no way to create new archive trigger based on new columns if the archive has columns that aren't in the

if (present && destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)).Any())//only mess about with column removal if there is an archive trigger
{

//move everything into the archive - do this by updating the HIC_validfrom
var sql = $"UPDATE {existing.GetFullyQualifiedName()} set {SpecialFieldNames.ValidFrom} = GETDATE()";
using var con = _destinationDatabase.Server.GetConnection();
con.Open();
using var cmd = _destinationDatabase.Server.GetCommand(sql, con);
cmd.CommandTimeout = 30000;
cmd.ExecuteNonQuery();

var removedColumns = destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c));
foreach (var column in removedColumns)
{
var discoveredColumn = existing.DiscoverColumn(column);
existing.DropColumn(discoveredColumn);
}

Check notice

Code scanning / CodeQL

Missed opportunity to use Select Note

This foreach loop immediately
maps its iteration variable to another variable
Loading
- consider mapping the sequence explicitly using '.Select(...)'.
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
string triggerProblems = "";
string triggerOK = "";
implementor.DropTrigger(out triggerProblems, out triggerOK);
if (triggerProblems != "")
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, triggerProblems));
}

existing = _destinationDatabase.ExpectTable(tblName);
implementor = triggerFactory.Create(existing);
try
{
present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled;
}
catch (TriggerMissingException)
{
present = false;
}
}
}

}
Expand All @@ -229,18 +293,6 @@
}
else if (UseArchiveTrigger && hasPKs)
{

TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer);
var implementor = triggerFactory.Create(existing);
bool present;
try
{
present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled;
}
catch (TriggerMissingException)
{
present = false;
}
//check the columns are correct, we might have added some
var existingColumns = existing.DiscoverColumns();
var existingColumnNames = existingColumns.Select(ec => ec.GetRuntimeName());
Expand All @@ -254,7 +306,10 @@
foreach (var column in newColumns)
{
existing.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000);
archiveTable.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000);
if (archiveTable.DiscoverColumns().All(col => col.GetRuntimeName() != column))
{
archiveTable.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000);
}
}
if (present)
{
Expand Down
45 changes: 35 additions & 10 deletions Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,23 +259,43 @@

//these were added during transaction so we have to specify them again here because transaction will not have been committed yet
sqlToRun = sqlToRun.Trim();
sqlToRun += $",{Environment.NewLine}";
sqlToRun += $"\thic_validTo datetime,{Environment.NewLine}";
sqlToRun += "\thic_userID varchar(128),";
sqlToRun += "\thic_status char(1)";
if (!sqlToRun.Contains("hic_validTo"))
{
sqlToRun += $"{Environment.NewLine}";
sqlToRun += $",\thic_validTo datetime";
}
if (!sqlToRun.Contains("hic_userID"))
{
sqlToRun += ",\thic_userID varchar(128)";
}
if (!sqlToRun.Contains("hic_status"))
{
sqlToRun += ",\thic_status char(1)";
}


sqlToRun += $"){Environment.NewLine}";
sqlToRun += $"AS{Environment.NewLine}";
sqlToRun += $"BEGIN{Environment.NewLine}";
sqlToRun += Environment.NewLine;

var liveCols = _columns.Select(c => $"[{c.GetRuntimeName()}]").Union(new string[]
{
$"[{SpecialFieldNames.DataLoadRunID}]", $"[{SpecialFieldNames.ValidFrom}]"
}).Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true).ToArray();

var archiveCols = $"{string.Join(",", liveCols)},hic_validTo,hic_userID,hic_status";
//var x = _archiveTable.DiscoverColumns();
//var liveCols = _columns.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]").Union(new string[]
var liveCols = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]")
//.Where(c => c != "[hic_validTo]" && c != "[hic_userID]" && c != "[hic_status]")
.ToList();

if (!liveCols.Contains($"[{SpecialFieldNames.DataLoadRunID}]")) liveCols.Add($"[{SpecialFieldNames.DataLoadRunID}]");
if (!liveCols.Contains($"[{SpecialFieldNames.ValidFrom}]")) liveCols.Add($"[{SpecialFieldNames.ValidFrom}]");
liveCols = liveCols.Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true).ToList();

Check notice

Code scanning / CodeQL

Unnecessarily complex Boolean expression Note

The expression 'A ? B : true' can be simplified to '!A || B'.
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed

var archiveCols = $"{string.Join(",", liveCols)}";
//$",hic_validTo,hic_userID,hic_status";
//if (!archiveCols.Contains("hic_validTo")) archiveCols += ",hic_validTo";
//if (!archiveCols.Contains("hic_userID")) archiveCols += ",hic_userID ";
//if (!archiveCols.Contains("hic_status")) archiveCols += ",hic_status ";

var cDotArchiveCols = string.Join(",", liveCols.Select(s => $"c.{s}"));


Expand All @@ -285,9 +305,12 @@
", '1899/01/01') AND hic_validTo" + Environment.NewLine, _archiveTable);
sqlToRun += Environment.NewLine;

var nullCoulmns = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]").Except(_table.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]"));
cDotArchiveCols = string.Join(",", liveCols.Select(s => nullCoulmns.Contains(s) ? $"NULL AS {s}" : $"c.{s}"));

sqlToRun += $"\tINSERT @returntable{Environment.NewLine}";
sqlToRun +=
$"\tSELECT {cDotArchiveCols},NULL AS hic_validTo, NULL AS hic_userID, 'C' AS hic_status{Environment.NewLine}"; //c is for current
$"\tSELECT {cDotArchiveCols}";//,NULL AS hic_validTo, NULL AS hic_userID, 'C' AS hic_status{Environment.NewLine}"; //c is for current
sqlToRun += string.Format("\tFROM [{0}] c" + Environment.NewLine, _table.GetRuntimeName());
sqlToRun += $"\tLEFT OUTER JOIN @returntable a ON {Environment.NewLine}";

Expand All @@ -300,6 +323,8 @@
sqlToRun += $"\tAND{Environment.NewLine}"; //add an AND because there are more coming
}

//TODO: ordering issue when adding a new column and removing an existing one

sqlToRun += string.Format("\tWHERE a.[{0}] IS NULL -- where archive record doesn't exist" + Environment.NewLine,
_primaryKeys.First().GetRuntimeName());
sqlToRun += $"\tAND @index > ISNULL(c.{SpecialFieldNames.ValidFrom}, '1899/01/01'){Environment.NewLine}";
Expand Down
5 changes: 3 additions & 2 deletions Rdmp.Core/DataLoad/Triggers/Implementations/TriggerImplementer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,10 @@ protected virtual void AddValidFrom(DiscoveredTable table, IQuerySyntaxHelper sy
private string WorkOutArchiveTableCreationSQL()
{
//script original table
var createTableSQL = _table.ScriptTableCreation(true, true, true);
var tbl = _archiveTable.Exists() ? _archiveTable : _table;
var createTableSQL = tbl.ScriptTableCreation(true, true, true);

var toReplaceTableName = $"CREATE TABLE {_table.GetFullyQualifiedName()}";
var toReplaceTableName = $"CREATE TABLE {tbl.GetFullyQualifiedName()}";

if (!createTableSQL.Contains(toReplaceTableName))
throw new Exception($"Expected to find occurrence of {toReplaceTableName} in the SQL {createTableSQL}");
Expand Down
Loading