diff --git a/DBTestCompareGenerator/Configuration.cs b/DBTestCompareGenerator/Configuration.cs index 0c629f8..1b5b0e5 100644 --- a/DBTestCompareGenerator/Configuration.cs +++ b/DBTestCompareGenerator/Configuration.cs @@ -34,6 +34,173 @@ public static string ConnectionString get { return Configuration.Builder["appSettings:ConnectionStrings:DB"]; } } + public static string DacpacFolder + { + get + { + string setting = null; + setting = Configuration.Builder["appSettings:DacpacFolder"]; + Logger.Trace(CultureInfo.CurrentCulture, "DacpacFolder Folder value from settings file '{0}'", setting); + return setting; + } + } + + public static string Database + { + get { return Configuration.Builder["appSettings:Database"]; } + } + + public static string FolderPath + { + get + { + string setting = null; + setting = Configuration.Builder["appSettings:Folder"]; + Logger.Trace(CultureInfo.CurrentCulture, "Folder value from settings file '{0}'", setting); + return setting; + } + } + + public static bool ExtractAllTableData + { + get + { + var setting = Configuration.Builder["appSettings:ExtractAllTableData"]; + Logger.Trace(CultureInfo.CurrentCulture, "Read ExtractAllTableData '{0}'", setting); + if (string.IsNullOrEmpty(setting)) + { + return false; + } + + if (setting.ToLower(CultureInfo.CurrentCulture).Equals("true")) + { + return true; + } + + return false; + } + } + + public static bool ExtractApplicationScopedObjectsOnly + { + get + { + var setting = Configuration.Builder["appSettings:ExtractApplicationScopedObjectsOnly"]; + Logger.Trace(CultureInfo.CurrentCulture, "Read ExtractApplicationScopedObjectsOnly '{0}'", setting); + if (string.IsNullOrEmpty(setting)) + { + return false; + } + + if (setting.ToLower(CultureInfo.CurrentCulture).Equals("true")) + { + return true; + } + + return false; + } + } + + public static bool VerifyExtraction + { + get + { + var setting = Configuration.Builder["appSettings:VerifyExtraction"]; + Logger.Trace(CultureInfo.CurrentCulture, "Read VerifyExtraction'{0}'", setting); + if (string.IsNullOrEmpty(setting)) + { + return false; + } + + if (setting.ToLower(CultureInfo.CurrentCulture).Equals("true")) + { + return true; + } + + return false; + } + } + + public static bool IgnoreExtendedProperties + { + get + { + var setting = Configuration.Builder["appSettings:IgnoreExtendedProperties"]; + Logger.Trace(CultureInfo.CurrentCulture, "Read IgnoreExtendedProperties '{0}'", setting); + if (string.IsNullOrEmpty(setting)) + { + return false; + } + + if (setting.ToLower(CultureInfo.CurrentCulture).Equals("true")) + { + return true; + } + + return false; + } + } + + public static bool IgnorePermissions + { + get + { + var setting = Configuration.Builder["appSettings:IgnorePermissions"]; + Logger.Trace(CultureInfo.CurrentCulture, "Read IgnorePermissions '{0}'", setting); + if (string.IsNullOrEmpty(setting)) + { + return false; + } + + if (setting.ToLower(CultureInfo.CurrentCulture).Equals("true")) + { + return true; + } + + return false; + } + } + + public static bool SaveAsBaseline + { + get + { + var setting = Configuration.Builder["appSettings:SaveAsBaseline"]; + Logger.Trace(CultureInfo.CurrentCulture, "Read IgnorePermissions '{0}'", setting); + if (string.IsNullOrEmpty(setting)) + { + return false; + } + + if (setting.ToLower(CultureInfo.CurrentCulture).Equals("true")) + { + return true; + } + + return false; + } + } + + public static bool UnpackDacpac + { + get + { + var setting = Configuration.Builder["appSettings:UnpackDacpac"]; + Logger.Trace(CultureInfo.CurrentCulture, "Read UnpackDacpac '{0}'", setting); + if (string.IsNullOrEmpty(setting)) + { + return false; + } + + if (setting.ToLower(CultureInfo.CurrentCulture).Equals("true")) + { + return true; + } + + return false; + } + } + public static List ColumnTypesToGroupBy { get diff --git a/DBTestCompareGenerator/DBTestCompareGenerator.csproj b/DBTestCompareGenerator/DBTestCompareGenerator.csproj index 91b2446..f395edb 100644 --- a/DBTestCompareGenerator/DBTestCompareGenerator.csproj +++ b/DBTestCompareGenerator/DBTestCompareGenerator.csproj @@ -34,6 +34,7 @@ + diff --git a/DBTestCompareGenerator/Program.cs b/DBTestCompareGenerator/Program.cs index 8e22fcf..1d35b99 100644 --- a/DBTestCompareGenerator/Program.cs +++ b/DBTestCompareGenerator/Program.cs @@ -8,6 +8,12 @@ internal static class Program { private static void Main(string[] args) { + if (Configuration.UnpackDacpac) + { + UnpackDacpac.ExtractDacpacFile(); + UnpackDacpac.UnpackDacpacFile(); + } + CopyConfigFiles.CopyConfigFile(); var configFromExcel = ReadConfigurationFromXlsx.ReadExcelFile(); CountQuerySqlServer.CreateCountQuery(configFromExcel); diff --git a/DBTestCompareGenerator/UnpackDacpac.cs b/DBTestCompareGenerator/UnpackDacpac.cs new file mode 100644 index 0000000..aa6c561 --- /dev/null +++ b/DBTestCompareGenerator/UnpackDacpac.cs @@ -0,0 +1,381 @@ +namespace DBTestCompareGenerator +{ + using System; + using System.IO; + using System.Linq; + using System.Text.RegularExpressions; + using Microsoft.SqlServer.Dac; + using Microsoft.SqlServer.Dac.Model; + + internal static class UnpackDacpac + { + private static readonly NLog.Logger Logger = NLog.Web.NLogBuilder.ConfigureNLog("nlog.config").GetCurrentClassLogger(); + + public static void ExtractDacpacFile() + { + string dacpacFilePath = $"{Configuration.DacpacFolder}"; + + if (!Directory.Exists(dacpacFilePath)) + { + Directory.CreateDirectory(dacpacFilePath); + Logger.Debug($"Folder created: {dacpacFilePath}"); + } + + string dacpackFile = $"{dacpacFilePath}{Path.DirectorySeparatorChar}{Configuration.Database}.dacpac"; + + Logger.Info($"Extracting dacpac file {dacpackFile} from DB {Configuration.Database}"); + + DacExtractOptions extractOptions = new DacExtractOptions + { + ExtractAllTableData = Configuration.ExtractAllTableData, + ExtractApplicationScopedObjectsOnly = Configuration.ExtractApplicationScopedObjectsOnly, + VerifyExtraction = Configuration.VerifyExtraction, + IgnoreExtendedProperties = Configuration.IgnoreExtendedProperties, + IgnorePermissions = Configuration.IgnorePermissions, + }; + + DacServices dacServices = new DacServices(Configuration.ConnectionString); + + dacServices.Message += (sender, e) => + { + Logger.Debug($"Message: {e.Message.MessageType} - {e.Message.Message}"); + }; + + dacServices.Extract(dacpackFile, Configuration.Database, "Mi9", new Version(), null, null, extractOptions); + } + + public static void UnpackDacpacFile() + { + var dacpacPath = $"{Configuration.DacpacFolder}{Path.DirectorySeparatorChar}{Configuration.Database}.dacpac"; + + Logger.Info($"Unpacking dacpac file {dacpacPath} from DB {Configuration.Database}"); + + var model = TSqlModel.LoadFromDacpac(dacpacPath, new ModelLoadOptions(DacSchemaModelStorageType.Memory, false)); + + foreach (var dbObject in model.GetObjects(DacQueryScopes.UserDefined).Reverse()) + { + string folder = $"{Configuration.FolderPath}{Path.DirectorySeparatorChar}{Configuration.Database}"; + string file = string.Empty; + + if (dbObject.TryGetScript(out var script)) + { + Logger.Info($"{dbObject.ObjectType.Name} - {dbObject.Name} processing"); + if (string.IsNullOrEmpty(dbObject.Name.ToString())) + { + file = $"{Configuration.Database}.Role"; + if (Configuration.SaveAsBaseline) + { + file = $"Role"; + } + + SaveTextToFile(script, folder, file); + } + else + { + // Get Schema name and Object Name + var fullName = dbObject.Name.Parts; + string type = dbObject.ObjectType.Name.ToString(); + + if (fullName.Count >= 4 && !type.Equals("Permission")) + { + // Get Object Name + file = $"{fullName[2]}"; + type = $"{fullName[0]}"; + type = AlterToType(type, file, script, out file); + + // Add DB Name and Schema Name to file Name + if (!Configuration.SaveAsBaseline) + { + file = $"{Configuration.Database}.{fullName[1]}.{file}"; + } + + folder = $"{folder}{Path.DirectorySeparatorChar}{fullName[1]}{Path.DirectorySeparatorChar}{SetType(type)}"; + } + else if ((fullName.Count == 3 && (type.Equals("Index") || type.Equals("ColumnStoreIndex"))) || fullName.Count == 2) + { + // Get Object Name + file = $"{fullName[1]}"; + + type = AlterToType(type, file, script, out file); + + // Add DB Name and Schema Name to file Name + if (!Configuration.SaveAsBaseline) + { + file = $"{Configuration.Database}.{fullName[0]}.{file}"; + } + + folder = $"{folder}{Path.DirectorySeparatorChar}{fullName[0]}{Path.DirectorySeparatorChar}{SetType(type)}"; + } + else if (type.Equals("Permission")) + { + // Get Object Name + file = $"{type}"; + + type = AlterToType(type, file, script, out file); + + // Add DB Name and Schema Name to file Name + if (!Configuration.SaveAsBaseline) + { + file = $"{Configuration.Database}.{file}"; + } + + folder = $"{folder}{Path.DirectorySeparatorChar}{fullName[2]}{Path.DirectorySeparatorChar}{SetType(type)}"; + } + else if (fullName.Count == 3) + { + // Get Object Name + file = $"{fullName[1]}"; + type = $"{fullName[0]}"; + type = AlterToType(type, file, script, out file); + + // Add DB Name and Schema Name to file Name + if (!Configuration.SaveAsBaseline) + { + file = $"{Configuration.Database}.{file}"; + } + + folder = $"{folder}{Path.DirectorySeparatorChar}{fullName[1]}{Path.DirectorySeparatorChar}{SetType(type)}"; + } + else + { + // Add DB Name and Schema Name to file Name + if (Configuration.SaveAsBaseline) + { + file = $"{type}"; + } + else + { + file = $"{Configuration.Database}.{type}"; + } + + // Add Schema Name and Object Type as Subfolders + folder = $"{folder}{Path.DirectorySeparatorChar}"; + } + + // Add Object type to file name + if (!Configuration.SaveAsBaseline) + { + file = $"{file}__{type}"; + } + + SaveTextToFile(script, folder, file); + } + } + else + Logger.Info($"{dbObject.ObjectType} - {dbObject.Name} could not be scripted"); + } + } + + static void SaveTextToFile(string text, string folder, string file) + { + if (string.IsNullOrEmpty(file)) + { + file = "ToBeChecked"; + } + + if (!Directory.Exists(folder)) + { + Directory.CreateDirectory(folder); + Logger.Debug($"Folder created: {folder}"); + } + + file = file.Replace("*", string.Empty).Replace($"{Path.DirectorySeparatorChar}", "_"); + file = $"{file}.sql"; + string fullPath = Path.Combine(folder, file); + if (File.Exists(fullPath)) + { + bool hasEmptyLineAtEnd = CheckForEmptyLineAtEnd(fullPath); + if (!hasEmptyLineAtEnd) + { + text = $"\n{text}"; + } + } + + Logger.Trace($"Saving script {file}"); + File.AppendAllText(Path.Combine(folder, file), text); + } + + static string SetType(string text) + { + if (text.Equals("TableValuedFunction")) + { + Logger.Trace($"Set type Functions"); + return "Functions"; + } + + if (text.Equals("Procedure")) + { + Logger.Trace($"Set type Procedure"); + return "Stored Procedures"; + } + + if (text.Equals("TableType")) + { + Logger.Trace($"Set type TableType"); + return "User Defined Types"; + } + + if (text.Equals("Table")) + { + Logger.Trace($"Set type Table"); + return "Tables"; + } + + if (text.Equals("View")) + { + Logger.Trace($"Set type View"); + return "Views"; + } + + if (text.Equals("ScalarFunction")) + { + Logger.Trace($"Set type Functions"); + return "Functions"; + } + + if (text.Equals("Sequence")) + { + Logger.Trace($"Set type Sequence"); + return "Sequences"; + } + + if (text.Equals("Synonym")) + { + Logger.Trace($"Set type Synonym"); + return "Synonyms"; + } + + return text; + } + + static string AlterToType(string text, string fileName, string script, out string file) + { + file = fileName; + if (text.Equals("PrimaryKeyConstraint")) + { + file = FindTableName(script, text); + text = text.Replace("PrimaryKeyConstraint", "Table"); + return text; + } + + if (text.Equals("DefaultConstraint")) + { + file = FindTableName(script, text); + text = text.Replace("DefaultConstraint", "Table"); + return text; + } + + if (text.Equals("CheckConstraint")) + { + file = FindTableName(script, text); + text = text.Replace("CheckConstraint", "Table"); + return text; + } + + if (text.Equals("ForeignKeyConstraint")) + { + file = FindTableName(script, text); + text = text.Replace("ForeignKeyConstraint", "Table"); + return text; + } + + if (text.Equals("UniqueConstraint")) + { + file = FindTableName(script, text); + text = text.Replace("UniqueConstraint", "Table"); + return text; + } + + if (text.Equals("ColumnStoreIndex")) + { + file = FindTableName(script, text); + text = text.Replace("ColumnStoreIndex", "Table"); + return text; + } + + if (text.Equals("DmlTrigger")) + { + file = FindTableName(script, text); + text = text.Replace("DmlTrigger", "Table"); + return text; + } + + if (text.Equals("Index")) + { + text = text.Replace("Index", "Table"); + return text; + } + + if (text.Equals("SqlView")) + { + text = text.Replace("SqlView", "View"); + return text; + } + + if (text.Equals("SqlTableBase")) + { + text = text.Replace("SqlTableBase", "Table"); + return text; + } + + if (text.Equals("SqlSubroutineParameter")) + { + text = text.Replace("SqlSubroutineParameter", "Procedure"); + return text; + } + + if (text.Equals("SqlProcedure")) + { + text = text.Replace("SqlProcedure", "Procedure"); + return text; + } + + if (text.Equals("SqlFunction")) + { + text = text.Replace("SqlFunction", "Functions"); + return text; + } + + if (text.Equals("SqlColumn")) + { + text = text.Replace("SqlColumn", "Table"); + return text; + } + + return text; + } + + static string FindTableName(string text, string type) + { + Regex r = new Regex(@"[\[]?[a-zA-Z_0-9]+[\]]?\.[\[]?([a-zA-Z_0-9]+)[\]]?"); + foreach (Match m in r.Matches(text)) + { + if (type.Equals("DmlTrigger")) + { + type = "Done"; + continue; + } + + return m.Groups[1].Value; + } + + return "NameOfTableNotFound"; + } + + static bool CheckForEmptyLineAtEnd(string filePath) + { + using (StreamReader reader = new StreamReader(filePath)) + { + string lastNonEmptyLine = null; + + while (reader.ReadLine() is { } line) + { + lastNonEmptyLine = line; + } + + // Check if the last non-empty line is not null and if it is empty + return string.IsNullOrWhiteSpace(lastNonEmptyLine); + } + } + } +} diff --git a/DBTestCompareGenerator/appsettings.json b/DBTestCompareGenerator/appsettings.json index eddce99..8db0338 100644 --- a/DBTestCompareGenerator/appsettings.json +++ b/DBTestCompareGenerator/appsettings.json @@ -7,8 +7,18 @@ "ColumnTypesToGroupBy": "nvarchar,nchar,datetime,date,bit", "DBNameLiveMinusTests": "AdventureWorks2008R2", "DBNameBranchMinusTests": "AdventureWorks2008R2", + "ExtractAllTableData": "false", + "ExtractApplicationScopedObjectsOnly": "false", + "VerifyExtraction": "false", + "IgnoreExtendedProperties": "false", + "IgnorePermissions": "false", + "SaveAsBaseline": "false", + "UnpackDacpac": "false", + "Database": "AdventureWorks2008R2", + "DacpacFolder": "Dacpac", + "Folder": "Definitions", "ConnectionStrings": { - "DB": "User ID=SA;Password=yourStrong22Password;Initial Catalog=AdventureWorks2008R2;Data Source=localhost;" + "DB": "User ID=SA;Password=yourStrong22Password;Initial Catalog=AdventureWorks2008R2;Data Source=localhost;TrustServerCertificate=True;" } } } \ No newline at end of file diff --git a/README.md b/README.md index e8ad8cf..f7f07af 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ # DBTestCompareGenerator ## Tool for generating database tests that can be run with **[DBTestCompare](https://github.com/ObjectivityLtd/DBTestCompare)** +### And extracting and unpacking DACPAC **Microsoft SQL Server DAC Package File** using [DacFx package](https://github.com/microsoft/DacFx). [![Build Status](https://dev.azure.com/DBTestCompare/Build/_apis/build/status/ObjectivityLtd.DBTestCompareGenerator)](https://dev.azure.com/DBTestCompare/Build/_build/latest?definitionId=2&_a=summary) [![Azure DevOps tests](https://img.shields.io/azure-devops/tests/DBTestCompare/Build/2?compact_message)](https://dev.azure.com/DBTestCompare/Build/_build?definitionId=2&_a=summary) @@ -165,3 +166,17 @@ If you set JAVA_HOME variable: java -jar DBTestCompare-1.0-SNAPSHOT-jar-with-dependencies.jar ``` More details can be found [here](https://github.com/ObjectivityLtd/DBTestCompare/wiki/Getting-started) + +### 7. To extract and unpack DACPAC **Microsoft SQL Server DAC Package File** change following settings in *appsettings.json* + +```json + "ExtractAllTableData": "false", + "ExtractApplicationScopedObjectsOnly": "false", + "VerifyExtraction": "false", + "IgnoreExtendedProperties": "false", + "IgnorePermissions": "false", + "SaveAsBaseline": "false", + "UnpackDacpac": "true", + "DacpacFolder": "c:\\ProjectPath\\Dacpac", + "Folder": "c:\\ProjectPath\\Definitions", +``` diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0214363..cc07d98 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -93,7 +93,6 @@ jobs: rm -rfv LICENSE ./restore-backup.ps1 - - task: CmdLine@2 displayName: set chmod run DBTestCompareGenerator inputs: @@ -104,7 +103,7 @@ jobs: chmod 777 ./DBTestCompare-*-SNAPSHOT-jar-with-dependencies.jar ls -alR ./DBTestCompareGenerator - + - task: PowerShell@2 displayName: run DBTestCompare inputs: @@ -126,12 +125,31 @@ jobs: targetType: 'inline' script: | cd .\DBTestCompareGenerator\bin\$(buildConfiguration)\net6.0 - .\set-appsettings.ps1 ".\" "appsettings.json" "appSettings" "ReadExcelFile" "true" $true + .\set-appsettings.ps1 ".\" "appsettings.json" "appSettings" "ReadExcelFile|DacpacFolder|Folder|UnpackDacpac" "true|$(Build.SourcesDirectory)\Dacpac|$(Build.SourcesDirectory)\Current|true" $true ./DBTestCompareGenerator .\set-tokens-for-tests.ps1 -OutDir ".\test-definitions\" -FileType "cmpSqlResults-config.xml" -token "\$\{SQL_SERVER\}|\$\{SQL_SERVERDBNAME\}|\$\{SQL_SERVER_USERNAME\}|\$\{SQL_SERVER_PASSWORD\}" -Value "$(SQL_SERVER)|$(SQL_SERVERDBNAME)|$(SQL_SERVER_USERNAME)|$(SQL_SERVER_PASSWORD)" $DBTestCompare = (Resolve-Path ".\DBTestCompare-*-SNAPSHOT-jar-with-dependencies.jar").ToString() /usr/bin/java -jar $DBTestCompare - + + - task: ArchiveFiles@2 + displayName: 'Zip generated objects definition' + inputs: + rootFolderOrFile: '$(Build.SourcesDirectory)\Current' + includeRootFolder: true + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.BuildId).zip' + replaceExistingArchive: true + - task: PublishBuildArtifacts@1 + displayName: 'Publish sql files' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/$(Build.BuildId).zip' + ArtifactName: 'definitions' + - task: PublishBuildArtifacts@1 + displayName: 'Publish dacpack files' + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\Dacpac/$(SQL_SERVERDBNAME).dacpac' + ArtifactName: '$(SQL_SERVERDBNAME)' + - task: PublishTestResults@2 inputs: testResultsFormat: 'JUnit'