diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs index 34d50b3d06..e0eb24dac4 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs @@ -36,6 +36,7 @@ using Tests.Common; using Tests.Common.Scenarios; using TypeGuesser; +using YamlDotNet.Serialization.NodeDeserializers; namespace Rdmp.Core.Tests.DataExport.DataExtraction; @@ -774,4 +775,1929 @@ public void SQLServerDestinationWithTriggersNoPKs() { Assert.That(dt.Rows, Has.Count.EqualTo(2)); } + + //add a column + [Test] + public void SQLServerDestinationWithTriggersAddAColumn() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 4"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //remove a column + [Test] + public void SQLServerDestinationWithTriggersRemoveAColumn() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 5"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //remove a column, add a different column + [Test] + public void SQLServerDestinationWithTriggersRemoveAColumnAddAColumn() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 6"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + + + var cols_1 = ec.GetAllExtractableColumnsFor(eds); + var col_1 = cols_1.First(c => c.SelectSQL.Contains("current_address_L4")); + var order = col_1.Order; + var selectSQL = col_1.SelectSQL; + var cei = col_1.CatalogueExtractionInformation; + col_1.DeleteInDatabase(); + + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + var cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + //col.DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(38)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + //cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + //col.DeleteInDatabase(); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //remove a column, add the column back + [Test] + public void SQLServerDestinationWithTriggersRemoveAColumnAddItBack() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 7"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + + + var cols_1 = ec.GetAllExtractableColumnsFor(eds); + var col_1 = cols_1.First(c => c.SelectSQL.Contains("current_address_L4")); + var order = col_1.Order; + var selectSQL = col_1.SelectSQL; + var cei = col_1.CatalogueExtractionInformation; + //col_1.DeleteInDatabase(); + + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + var cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + //cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + //col.DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + cols = ec.GetAllExtractableColumnsFor(eds); + //cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + //cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + //col.DeleteInDatabase(); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //add a column, remove a different column + [Test] + public void SQLServerDestinationWithTriggersAddAColumnThenRemoveADifferentOne() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 8"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + //current_address_L4 + cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(3)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //add a column, remove the same column + [Test] + public void SQLServerDestinationWithTriggersAddAColumnThenRemoveTheSameOne() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 9"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + //current_address_L4 + cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + + //add a column and remove a column + [Test] + public void SQLServerDestinationWithTriggersAddAColumnAndRemoveAtSameTime() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 10"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } } \ No newline at end of file diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/MSSQLMergeDestinationTests.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/MSSQLMergeDestinationTests.cs index 68dc5e53e8..10bfb0e502 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/MSSQLMergeDestinationTests.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/MSSQLMergeDestinationTests.cs @@ -1,6 +1,16 @@ using FAnsi; +using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; using NUnit.Framework; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands.CatalogueCreationCommands; +using Rdmp.Core.CommandExecution.AtomicCommands.CohortCreationCommands; +using Rdmp.Core.CommandLine.DatabaseCreation; +using Rdmp.Core.CommandLine.Options; +using Rdmp.Core.CommandLine.Runners; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Aggregation; +using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.Curation.Data.Pipelines; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction.Commands; @@ -10,15 +20,21 @@ using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.QueryBuilding; +using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; +using SynthEHR; +using SynthEHR.Datasets; using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; using System.Data; +using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using Tests.Common; using Tests.Common.Scenarios; +using TypeGuesser; namespace Rdmp.Core.Tests.DataExport.DataExtraction { @@ -31,7 +47,7 @@ public void Execute(DataTable dt) { } } - public class MSSQLMergeDestinationTests: TestsRequiringAnExtractionConfiguration + public class MSSqlMergeDestinationTests: TestsRequiringAnExtractionConfiguration { //create table first time [Test] @@ -203,5 +219,1954 @@ public void MSSQLMerge_Merge_Delete() Assert.That(tbl.GetDataTable().Rows[0].ItemArray, Is.EqualTo(new List() {2, "two" })); tbl.Drop(); } + + + private FileInfo CreateFileInForLoading(string filename, int rows, Random r) + { + var fi = new FileInfo(Path.Combine(Path.GetTempPath(), Path.GetFileName(filename))); + + var demog = new Demography(r); + var people = new PersonCollection(); + people.GeneratePeople(500, r); + + demog.GenerateTestDataFile(people, fi, rows); + + return fi; + } + + //add a column + [Test] + public void MSSQLMerge_SQLServerDestinationWithTriggersAddAColumn() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 4"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(MSSqlMergeDestination), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var DeleteMergeTempTable = destinationArguments.Single(a => a.Name == "DeleteMergeTempTable"); + + ////var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + DeleteMergeTempTable.SetValue(false); + DeleteMergeTempTable.SaveToDatabase(); + //reExtract.SetValue(true); + //reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + var nec = new ExtractableColumn(DataExportRepository,eds,ec,col.CatalogueExtractionInformation,0,col.SelectSQL); + nec.SaveToDatabase(); + ec.AddDatasetToConfiguration(eds); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //remove a column + [Test] + public void MSSQLMerge_SQLServerDestinationWithTriggersRemoveAColumn() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 5"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(MSSqlMergeDestination), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + //var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + //reExtract.SetValue(true); + //reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //remove a column, add a different column + [Test] + public void MSSQLMerge_SQLServerDestinationWithTriggersRemoveAColumnAddAColumn() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 6"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(MSSqlMergeDestination), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + //var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + //reExtract.SetValue(true); + //reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + + + var cols_1 = ec.GetAllExtractableColumnsFor(eds); + var col_1 = cols_1.First(c => c.SelectSQL.Contains("current_address_L3")); + var order = col_1.Order; + var selectSQL = col_1.SelectSQL; + var cei = col_1.CatalogueExtractionInformation; + col_1.DeleteInDatabase(); + + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + var cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + cols.First(c => c.SelectSQL.Contains("current_address_L3")).DeleteInDatabase(); + //col.DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(38)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + //cols.First(c => c.SelectSQL.Contains("current_address_L3")).DeleteInDatabase(); + //col.DeleteInDatabase(); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //remove a column, add the column back + [Test] + public void MSSQLMerge_SQLServerDestinationWithTriggersRemoveAColumnAddItBack() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 7"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(MSSqlMergeDestination), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + //var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + //reExtract.SetValue(true); + //reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + + + var cols_1 = ec.GetAllExtractableColumnsFor(eds); + var col_1 = cols_1.First(c => c.SelectSQL.Contains("current_address_L3")); + var order = col_1.Order; + var selectSQL = col_1.SelectSQL; + var cei = col_1.CatalogueExtractionInformation; + //col_1.DeleteInDatabase(); + + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + var cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + //cols.First(c => c.SelectSQL.Contains("current_address_L3")).DeleteInDatabase(); + //col.DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + cols = ec.GetAllExtractableColumnsFor(eds); + //cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + //cols.First(c => c.SelectSQL.Contains("current_address_L3")).DeleteInDatabase(); + //col.DeleteInDatabase(); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //add a column, remove a different column + [Test] + public void MSSQLMerge_SQLServerDestinationWithTriggersAddAColumnThenRemoveADifferentOne() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 8"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(MSSqlMergeDestination), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + //var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + //reExtract.SetValue(true); + //reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + //current_address_L3 + cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_address_L3")).DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //add a column, remove the same column + [Test] + public void MSSQLMerge_SQLServerDestinationWithTriggersAddAColumnThenRemoveTheSameOne() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 9"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(MSSqlMergeDestination), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + //var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + //reExtract.SetValue(true); + //reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + //current_address_L3 + cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + + //add a column and remove a column + [Test] + public void MSSQLMerge_SQLServerDestinationWithTriggersAddAColumnAndRemoveAtSameTime() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 10"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(MSSqlMergeDestination), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + //var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + //reExtract.SetValue(true); + //reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + cols.First(c => c.SelectSQL.Contains("current_address_L3")).DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + } } diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index 9c3f4f00e8..bf98423dfb 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -4,7 +4,6 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using Amazon.Auth.AccessControlPolicy; using FAnsi.Discovery; using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; @@ -14,8 +13,8 @@ using Rdmp.Core.DataExport.DataRelease.Pipeline; using Rdmp.Core.DataExport.DataRelease.Potential; using Rdmp.Core.DataFlowPipeline; -using Rdmp.Core.DataLoad.Engine.Job.Scheduling; using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; +using Rdmp.Core.DataLoad.Triggers; using Rdmp.Core.DataLoad.Triggers.Exceptions; using Rdmp.Core.DataLoad.Triggers.Implementations; using Rdmp.Core.MapsDirectlyToDatabaseTable; @@ -31,7 +30,6 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; -using YamlDotNet.Core; namespace Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; @@ -169,6 +167,15 @@ protected override void WriteRows(DataTable toProcess, IDataLoadEventListener jo LinesWritten += toProcess.Rows.Count; } + + private bool hasStructuralChanges(DataTable source, DiscoveredTable destination) + { + var sourceColumns = source.Columns.Cast().Select(c => c.ColumnName).ToList(); + var destinationColumns = destination.DiscoverColumns().Select(c => c.GetRuntimeName()).ToList(); + return !sourceColumns.All(destinationColumns.Contains) || !destinationColumns.All(sourceColumns.Contains); + } + + private DataTableUploadDestination PrepareDestination(IDataLoadEventListener listener, DataTable toProcess) { //see if the user has entered an extraction server/database @@ -191,7 +198,17 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis if (existing.Exists()) { var hasPKs = existing.DiscoverColumns().Any(col => col.IsPrimaryKey); - + TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer); + var implementor = triggerFactory.Create(existing); + bool triggerPresent; + try + { + triggerPresent = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + } + catch (TriggerMissingException) + { + triggerPresent = false; + } if (!AlwaysDropExtractionTables) { //check the PKs are the same @@ -207,6 +224,47 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis """)); return null; } + if (hasStructuralChanges(toProcess, existing)) + { + var sourceColumns = toProcess.Columns.Cast().Select(c => c.ColumnName).ToList(); + var destinationColumns = existing.DiscoverColumns().Select(c => c.GetRuntimeName()).ToList(); + + if (triggerPresent && destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)).Any())//only mess about with column removal if there is an archive trigger + { + + //move everything into the archive - do this by updating the HIC_validfrom + var sql = $"UPDATE {existing.GetFullyQualifiedName()} set {SpecialFieldNames.ValidFrom} = GETDATE()"; + using var con = _destinationDatabase.Server.GetConnection(); + con.Open(); + using var cmd = _destinationDatabase.Server.GetCommand(sql, con); + cmd.CommandTimeout = 30000; + cmd.ExecuteNonQuery(); + + var removedColumns = destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)); + foreach (var column in removedColumns.Select(c => existing.DiscoverColumn(c))) + { + existing.DropColumn(column); + } + string triggerProblems = ""; + string triggerOK = ""; + implementor.DropTrigger(out triggerProblems, out triggerOK); + if (triggerProblems != "") + { + listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, triggerProblems)); + } + + existing = _destinationDatabase.ExpectTable(tblName); + implementor = triggerFactory.Create(existing); + try + { + triggerPresent = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + } + catch (TriggerMissingException) + { + triggerPresent = false; + } + } + } } if (_request.IsBatchResume) @@ -228,18 +286,6 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis } else if (UseArchiveTrigger && hasPKs) { - - TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer); - var implementor = triggerFactory.Create(existing); - bool present; - try - { - present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; - } - catch (TriggerMissingException) - { - present = false; - } //check the columns are correct, we might have added some var existingColumns = existing.DiscoverColumns(); var existingColumnNames = existingColumns.Select(ec => ec.GetRuntimeName()); @@ -253,9 +299,12 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis foreach (var column in newColumns) { existing.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000); - archiveTable.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000); + if (archiveTable.DiscoverColumns().All(col => col.GetRuntimeName() != column)) + { + archiveTable.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000); + } } - if (present) + if (triggerPresent) { string triggerProblems = ""; string triggerOK = ""; @@ -267,12 +316,12 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis existing = _destinationDatabase.ExpectTable(tblName); implementor = triggerFactory.Create(existing); - present = false; + triggerPresent = false; } } } - if (!present) + if (!triggerPresent) { implementor.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet); } diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/MSSqlMergeDestination.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/MSSqlMergeDestination.cs index d3fafde7d3..83c307df83 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/MSSqlMergeDestination.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/MSSqlMergeDestination.cs @@ -80,6 +80,13 @@ public MSSqlMergeDestination() : base(false) { } + private bool hasStructuralChanges(DataTable source, DiscoveredTable destination) + { + var sourceColumns = source.Columns.Cast().Select(c => c.ColumnName).ToList(); + var destinationColumns = destination.DiscoverColumns().Select(c => c.GetRuntimeName()).ToList(); + return !sourceColumns.All(destinationColumns.Contains) || !destinationColumns.All(sourceColumns.Contains); + } + private string GetTableName(string suffix, DataTable dt) { string tblName = TableNamingPattern; @@ -136,7 +143,7 @@ public override void Abort(IDataLoadEventListener listener) - public static string GetMergeSQL(string destinationTableName,string tempTableName, DataColumn[] pkColumns, DataColumn[] nonPkColumns, bool performDeletes,IQuerySyntaxHelper syntaxHelper) + public static string GetMergeSQL(string destinationTableName, string tempTableName, DataColumn[] pkColumns, DataColumn[] nonPkColumns, bool performDeletes, IQuerySyntaxHelper syntaxHelper) { List columns = new(); columns.AddRange(pkColumns); @@ -230,6 +237,94 @@ protected override void Open(DataTable toProcess, IDataLoadEventListener job, Gr Destination PKs: {string.Join(", ", remotePKs)} """); } + TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer); + var implementor = triggerFactory.Create(existing); + bool triggerPresent; + try + { + triggerPresent = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + } + catch (TriggerMissingException) + { + triggerPresent = false; + } + if (hasStructuralChanges(toProcess, existing)) + { + var sourceColumns = toProcess.Columns.Cast().Select(c => c.ColumnName).ToList(); + var destinationColumns = existing.DiscoverColumns().Select(c => c.GetRuntimeName()).ToList(); + + if (triggerPresent && destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)).Any())//only mess about with column removal if there is an archive trigger + { + + //move everything into the archive - do this by updating the HIC_validfrom + var sql = $"UPDATE {existing.GetFullyQualifiedName()} set {SpecialFieldNames.ValidFrom} = GETDATE()"; + using var con = targetDb.Server.GetConnection(); + con.Open(); + using var cmd = targetDb.Server.GetCommand(sql, con); + cmd.CommandTimeout = 30000; + cmd.ExecuteNonQuery(); + + var removedColumns = destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)); + foreach (var column in removedColumns.Select(c => existing.DiscoverColumn(c))) + { + existing.DropColumn(column); + } + string triggerProblems = ""; + string triggerOK = ""; + implementor.DropTrigger(out triggerProblems, out triggerOK); + if (triggerProblems != "") + { + throw new Exception(triggerProblems); + } + + existing = targetDb.ExpectTable(tblName); + implementor = triggerFactory.Create(existing); + try + { + triggerPresent = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + } + catch (TriggerMissingException) + { + triggerPresent = false; + } + } + if (triggerPresent && sourceColumns.Except(destinationColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)).Any()) + { + var addedColumns = sourceColumns.Except(destinationColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)); + var archiveTable = existing.Database.ExpectTable(tblName + "_Archive"); + foreach (var column in addedColumns) + { + var colType = toProcess.Columns[column].DataType; + existing.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(colType), true, 30000); + if (archiveTable.Exists() && archiveTable.DiscoverColumns().All(col => col.GetRuntimeName() != column)) + { + archiveTable.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000); + } + } + string triggerProblems = ""; + string triggerOK = ""; + implementor.DropTrigger(out triggerProblems, out triggerOK); + if (triggerProblems != "") + { + throw new Exception(triggerProblems); + } + + existing = targetDb.ExpectTable(tblName); + implementor = triggerFactory.Create(existing); + try + { + triggerPresent = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + } + catch (TriggerMissingException) + { + triggerPresent = false; + } + } + } + if (!triggerPresent && UseArchiveTrigger) + { + implementor.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet); + } } } _toProcess = toProcess; @@ -287,7 +382,7 @@ protected override void WriteRows(DataTable toProcess, IDataLoadEventListener jo if (UseArchiveTrigger) { TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer); - var implementor = triggerFactory.Create(destinationTable, false, true); + var implementor = triggerFactory.Create(destinationTable); bool present; try { @@ -299,7 +394,14 @@ protected override void WriteRows(DataTable toProcess, IDataLoadEventListener jo } if (!present) { - implementor.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet); + try + { + implementor.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet); + } + catch (Exception e) + { + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, $"Failed to create archive trigger on {destinationTable.GetFullyQualifiedName()}: {e.Message}")); + } } } diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs index a4eb823784..c94eb8d569 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs @@ -259,10 +259,22 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch //these were added during transaction so we have to specify them again here because transaction will not have been committed yet sqlToRun = sqlToRun.Trim(); - sqlToRun += $",{Environment.NewLine}"; - sqlToRun += $"\thic_validTo datetime,{Environment.NewLine}"; - sqlToRun += "\thic_userID varchar(128),"; - sqlToRun += "\thic_status char(1)"; + if (!sqlToRun.Contains("hic_validTo")) + { + sqlToRun += $",{Environment.NewLine}"; + sqlToRun += $"hic_validTo datetime"; + } + if (!sqlToRun.Contains("hic_userID")) + { + sqlToRun += $",{Environment.NewLine}"; + + sqlToRun += "hic_userID varchar(128)"; + } + if (!sqlToRun.Contains("hic_status")) + { + sqlToRun += $",{Environment.NewLine}"; + sqlToRun += "hic_status char(1)"; + } sqlToRun += $"){Environment.NewLine}"; @@ -270,12 +282,16 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun += $"BEGIN{Environment.NewLine}"; sqlToRun += Environment.NewLine; - var liveCols = _columns.Select(c => $"[{c.GetRuntimeName()}]").Union(new string[] - { - $"[{SpecialFieldNames.DataLoadRunID}]", $"[{SpecialFieldNames.ValidFrom}]" - }).Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true).ToArray(); - var archiveCols = $"{string.Join(",", liveCols)},hic_validTo,hic_userID,hic_status"; + var liveCols = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]") + .ToList(); + + if (!liveCols.Contains($"[{SpecialFieldNames.DataLoadRunID}]")) liveCols.Add($"[{SpecialFieldNames.DataLoadRunID}]"); + if (!liveCols.Contains($"[{SpecialFieldNames.ValidFrom}]")) liveCols.Add($"[{SpecialFieldNames.ValidFrom}]"); + liveCols = liveCols.Where(col => !_dontAddDataLoadRunId || col != $"[{SpecialFieldNames.DataLoadRunID}]").ToList(); + + var archiveCols = $"{string.Join(",", liveCols)}"; + var cDotArchiveCols = string.Join(",", liveCols.Select(s => $"c.{s}")); @@ -285,9 +301,14 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch ", '1899/01/01') AND hic_validTo" + Environment.NewLine, _archiveTable); sqlToRun += Environment.NewLine; + //these are columns that exist in the archive, but not the table. So have to be set as NULL as + var nullColumns = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]").Except(_table.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]")).ToList(); + + cDotArchiveCols = string.Join(",", liveCols.Select(s => nullColumns.Contains(s) ? $"NULL AS {s}" : $"c.{s}")); + sqlToRun += $"\tINSERT @returntable{Environment.NewLine}"; sqlToRun += - $"\tSELECT {cDotArchiveCols},NULL AS hic_validTo, NULL AS hic_userID, 'C' AS hic_status{Environment.NewLine}"; //c is for current + $"\tSELECT {cDotArchiveCols}"; sqlToRun += string.Format("\tFROM [{0}] c" + Environment.NewLine, _table.GetRuntimeName()); sqlToRun += $"\tLEFT OUTER JOIN @returntable a ON {Environment.NewLine}"; diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/TriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/TriggerImplementer.cs index 95124fc55d..5b7313a08a 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/TriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/TriggerImplementer.cs @@ -136,9 +136,10 @@ protected virtual void AddValidFrom(DiscoveredTable table, IQuerySyntaxHelper sy private string WorkOutArchiveTableCreationSQL() { //script original table - var createTableSQL = _table.ScriptTableCreation(true, true, true); + var tbl = _archiveTable.Exists() ? _archiveTable : _table; + var createTableSQL = tbl.ScriptTableCreation(true, true, true); - var toReplaceTableName = $"CREATE TABLE {_table.GetFullyQualifiedName()}"; + var toReplaceTableName = $"CREATE TABLE {tbl.GetFullyQualifiedName()}"; if (!createTableSQL.Contains(toReplaceTableName)) throw new Exception($"Expected to find occurrence of {toReplaceTableName} in the SQL {createTableSQL}");