Merge pull request #290 from Azure-Player/bug-289

## [1.3.0] - 2023-02-27 ### Fixed * Defined ignored types in references and added `BigDataPool` to it #289
Azure-Player · Feb 27, 2023 · ae8d1b7 · ae8d1b7
2 parents 8e60f03 + a035b61
commit ae8d1b7
Show file tree

Hide file tree

Showing 9 changed files with 39 additions and 18 deletions.
diff --git a/README.md b/README.md
@@ -22,17 +22,17 @@ The main advantage of the module is the ability to publish all the Azure Data Fa
 * Built-in mechanism to replace, remove or add the properties with the indicated values (CSV and JSON file formats supported)
 * Stopping/starting triggers
 * Dropping objects when not exist in the source (code)  
-  * (new!) Optionally can skip deletion of excluded objects
-* Filtering (include or exclude) objects to be deployed by name and/or type **and/or folder** (new!)
+  * Optionally can skip deletion of excluded objects
+* Filtering (include or exclude) objects to be deployed by name and/or type **and/or folder**
 * Filtering supports wildcards
 * Publish options allow you to control:
   * Whether stop and restarting triggers
   * Whether delete or not objects not in the source
   * Whether create or not a new instance of ADF if it not exist
 * Tokenisation in config file allows replace any value by Environment Variable or Variable from DevOps Pipeline
-  * (new!) Allows to define multiple file (objects) by wildcarding
+  * Allows to define multiple file (objects) by wildcarding
 * Global Parameters
-* Support for Managed VNET and Managed Private Endpoint (new!)
+* Support for Managed VNET and Managed Private Endpoint 
 * Build function to support validation of files, dependencies and config
 * Test connections (Linked Services)
 * Generates mermaid dependencies diagram to be used in MarkDown type of documents

diff --git a/azure.datafactory.tools.psd1 b/azure.datafactory.tools.psd1
@@ -12,7 +12,7 @@
     RootModule = 'azure.datafactory.tools.psm1'
 
     # Version number of this module.
-    ModuleVersion = '1.2.0'
+    ModuleVersion = '1.3.0'
 
     # Supported PSEditions
     # CompatiblePSEditions = @()

diff --git a/changelog.md b/changelog.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.3.0] - 2023-02-27
+### Fixed
+* Defined ignored types in references and added `BigDataPool` to it #289
+
 ## [1.2.0] - 2023-02-24
 ### Fixed
 * Validation of ADF files fails when JSON contains property with single quote #287

diff --git a/private/AdfObject.class.ps1 b/private/AdfObject.class.ps1
@@ -83,11 +83,13 @@ class AdfObject {
         return $ofn
     }
 
-    static $AllowedTypes = @('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'factory', 'managedVirtualNetwork', 'managedPrivateEndpoint', 'credential', 'notebook')
+    static $AllowedTypes = @('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'factory', 'managedVirtualNetwork', 'managedPrivateEndpoint', 'credential')
+    static $IgnoreTypes  = @('notebook', 'BigDataPool')
 
     static AssertType ([string] $Type)
     {
-        if ($Type -notin [AdfObject]::allowedTypes ) { 
+        $AllTypes = [AdfObject]::AllowedTypes + [AdfObject]::IgnoreTypes
+        if ($Type -notin $AllTypes ) { 
             throw "ADFT0029: Unknown object type: $Type."
         }
     }

diff --git a/private/Deploy-AdfObject.ps1 b/private/Deploy-AdfObject.ps1
@@ -33,7 +33,7 @@ function Deploy-AdfObject {
                 } else {
                     Write-Error "ADFT0005: Referenced object [$type].[$name] was not found."
                 }
-            } elseif ($type -ne "notebook") {
+            } elseif ($type -notin [AdfObject]::IgnoreTypes) {
                 Deploy-AdfObject -obj $depobj
             }
             $i++

diff --git a/private/Get-AdfObjectByName.ps1 b/private/Get-AdfObjectByName.ps1
@@ -11,6 +11,14 @@ function Get-AdfObjectByName {
     $simtype = Get-SimplifiedType -Type "$type"
     switch -Exact ($simtype)
     {
+        {$simtype -in [AdfObject]::IgnoreTypes}
+        {
+            $r = New-Object -TypeName AdfObject 
+            $r.Name = "Ignored"
+            $r.Type = $simtype
+            $r.Adf = $adf
+            Break
+        }
         'IntegrationRuntime'
         {
             $r = $adf.IntegrationRuntimes | Where-Object { $_.Name -eq $name } | Select-Object -First 1
@@ -39,13 +47,6 @@ function Get-AdfObjectByName {
         {
             $r = $adf.Credentials | Where-Object { $_.Name -eq $name } | Select-Object -First 1
         }
-        'Notebook'
-        {
-            $r = New-Object -TypeName AdfObject 
-            $r.Name = "Null"
-            $r.Type = $simtype
-            $r.Adf = $adf
-        }
         'Factory'
         {
             $r = $adf.Factories | Where-Object { $_.Name -eq $name } | Select-Object -First 1

diff --git a/public/Test-AdfCode.ps1 b/public/Test-AdfCode.ps1
@@ -66,7 +66,8 @@ function Test-AdfCode {
             $_.DependsOn | ForEach-Object {
                 Write-Verbose -Message "  - Checking dependency: [$_]"
                 $ref_arr = $adf.GetObjectsByFullName("$_")
-                if ($ref_arr.Count -eq 0 -and $_ -notlike "notebook.*") {
+                $refName = [AdfObjectName]::new($_)
+                if ($ref_arr.Count -eq 0 -and $refName.Type -notin [AdfObject]::IgnoreTypes) {
                     $result.ErrorCount += 1
                     Write-Error -Message "Couldn't find referenced object $_." -ErrorAction 'Continue'
                 }

diff --git a/test/Get-ReferencedObjects.Tests.ps1 b/test/Get-ReferencedObjects.Tests.ps1
@@ -42,7 +42,7 @@ InModuleScope azure.datafactory.tools {
                 @{ Adf = 'BigFactorySample2'; Name = 'pipeline\TaxiDemo'; RefCount = 1},
                 @{ Adf = 'BigFactorySample2'; Name = 'dataflow\Currency Converter'; RefCount = 4},
                 @{ Adf = 'adf2';              Name = 'dataset\DS_Json'; RefCount = 1},
-                @{ Adf = 'adf2';              Name = 'pipeline\SynapseNotebook1'; RefCount = 2}
+                @{ Adf = 'adf2';              Name = 'pipeline\SynapseNotebook1'; RefCount = 3}
 
         It 'Should find <RefCount> refs in object "<Adf>\<Name>"' -TestCases $cases {
             param

diff --git a/test/adf2/pipeline/SynapseNotebook1.json b/test/adf2/pipeline/SynapseNotebook1.json
@@ -23,7 +23,20 @@
                         "type": "NotebookReference"
                     },
                     "snapshot": true,
-                    "Single'Quote": true
+                    "Single'Quote": true,
+                    "sparkPool": {
+                        "referenceName": {
+                            "value": "cosdsdsd",
+                            "type": "Expression"
+                        },
+                        "type": "BigDataPoolReference"
+                    },
+                    "conf": {
+                        "spark.dynamicAllocation.enabled": null,
+                        "spark.dynamicAllocation.minExecutors": null,
+                        "spark.dynamicAllocation.maxExecutors": null
+                    },
+                    "numExecutors": null
                 },
                 "linkedServiceName": {
                     "referenceName": "LS_AzureSynapseArtifacts1",