v2.4.23

delphix · Apr 24, 2024 · 0cc6c6a · 0cc6c6a
2 parents 92c6896 + 2f79ecd
commit 0cc6c6a
Show file tree

Hide file tree

Showing 14 changed files with 1,131 additions and 53 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 2.4.23
+
+### Added
+- Support for Postgresql ( dSource / VDB )
+- Data patch flag added for Oracle VDBs
+
 ## 2.4.22.1
 
 ### Changes

diff --git a/bin/dx_ctl_dsource.pl b/bin/dx_ctl_dsource.pl
@@ -89,6 +89,17 @@
   'uniqname=s'  => \(my $uniqname),
   'template=s' => \(my $template),
   'oracledbtype=s' => \(my $oracledbtype),
+  'sourcehostname=s' => \(my $sourcehostname),
+  'sourceport=n' => \(my $sourceport),
+  'ingestiontype=s' => \(my $ingestiontype),
+  'singledbname=s' => \(my $singledbname),
+  'stagingport=n' => \(my $stagingport),
+  'dumpdir=s' => \(my $dumpdir),
+  'restorejobs=n' => \(my $restorejobs),
+  'dumpjobs=n' => \(my $dumpjobs),
+  'customparameters=s@' => \(my $customparameters),
+  'backup_dir_log=s' => \(my $backup_dir_log),
+  'keepinsync=s' => \(my $keepinsync),
   'dever=s' => \(my $dever),
   'debug:n' => \(my $debug),
   'all' => (\my $all),
@@ -134,13 +145,13 @@
     exit (1);
   }
 
-  if ( defined ($type) && ( ! ( ( lc $type eq 'oracle') || ( lc $type eq 'sybase') || ( lc $type eq 'mssql') || ( lc $type eq 'vfiles') || ( lc $type eq 'db2') ) ) ) {
+  if ( defined ($type) && ( ! ( ( lc $type eq 'oracle') || ( lc $type eq 'sybase') || ( lc $type eq 'mssql') || ( lc $type eq 'vfiles') || ( lc $type eq 'db2') || ( lc $type eq 'postgresql') ) ) ) {
     print "Option -type has invalid parameter - $type \n";
     pod2usage(-verbose => 1,  -input=>\*DATA);
     exit (1);
   }
 
-  if (((lc $type eq 'vfiles') || (lc $type eq 'db2')) && (lc $action eq 'attach')) {
+  if (((lc $type eq 'vfiles') || ( lc $type eq 'db2') || ( lc $type eq 'postgresql')) && (lc $action eq 'attach')) {
     print "Can't attach $type dSource\n";
     exit (1);
   }
@@ -153,7 +164,7 @@
       exit (1);
     }
   }
-  elsif ( ( lc $type ne 'db2' ) && ( ! ( defined($type) && defined($sourcename) && defined($dsourcename)  && defined($source_os_user) && defined($group) ) ) )  {
+  elsif ( ( lc $type ne 'postgresql' ) && ( lc $type ne 'db2' ) && ( ! ( defined($type) && defined($sourcename) && defined($dsourcename)  && defined($source_os_user) && defined($group) ) ) )  {
     print "Options -sourcename, -dsourcename, -group, -source_os_user are required. \n";
     pod2usage(-verbose => 1,  -input=>\*DATA);
     exit (1);
@@ -175,7 +186,7 @@
 
 
 } else {
-  if (defined ($type) && ((lc $type eq 'vfiles') || (lc $type eq 'db2') ) && (lc $action eq 'detach')) {
+  if (defined ($type) && ((lc $type eq 'vfiles') || (lc $type eq 'db2') || ( lc $type eq 'postgresql') ) && (lc $action eq 'detach')) {
     print "Can't deattach $type dSource\n";
     exit (1);
   }
@@ -229,6 +240,12 @@
             pod2usage(-verbose => 1,  -input=>\*DATA);
             exit (1)
           }
+      } elsif (lc $type eq 'postgresql') {
+        if ( lc $ingestiontype ne 'externalbackup' ) {
+          print "Options -dbuser and -password are required for non vFiles dsources. \n";
+          pod2usage(-verbose => 1,  -input=>\*DATA);
+          exit (1);   
+        }
       } else {
         print "Options -dbuser and -password are required for non vFiles dsources. \n";
         pod2usage(-verbose => 1,  -input=>\*DATA);
@@ -410,6 +427,17 @@
       }
       $jobno = $db->addSource($sourcename,$sourceinst,$sourceenv,$source_os_user,$dbuser,$password,$dsourcename,$group,$logsync,$stageenv,$stageinst,$stage_os_user, $backup_dir, $hadr);
     }
+    elsif ($type eq 'postgresql') {
+      my $db = new PostgresVDB_obj($engine_obj,$debug);
+      if (addhooks($hooks, $db, $presync, $postsync)) {
+        $ret = $ret + 1;
+        last;
+      }
+      $jobno = $db->addSource($sourcename,$dbuser,$password,$dsourcename,$group,$logsync,$stageenv,$stageinst,$stage_os_user, $backup_dir, 
+               $sourcehostname, $sourceport, $ingestiontype, $dumpdir, $restorejobs, $dumpjobs, $stagingport, $singledbname, $mountbase,
+               $customparameters, $backup_dir, $backup_dir_log, $keepinsync);
+
+    }
 
     # we are adding only one dSource - so one job
     $ret = $ret + Toolkit_helpers::waitForAction($engine_obj, $jobno, "Action completed with success", "There were problems with dSource action");
@@ -508,6 +536,17 @@ =head1 SYNOPSIS
                 [-postsync [hookname,]template|filename[,OS_shell] ]
                 [-stagingpush]
                 [-oracledbtype nonmt|cdb|pdb]
+                [-customparameters (param_name=value)|(#param_name)]
+                [-ingestiontype single|initalized|externalbackup ] 
+                [-sourcehostname hostname ] 
+                [-sourceport xxx]
+                [-singledbname dbname]
+                [-stagingport xxx ]
+                [-dumpdir directory]
+                [-restorejobs x]
+                [-dumpjobs x]
+                [-backup_dir_log directory]
+                [-keepinsync yes|no]
                 [-debug ]
                 [-version ]
                 [-help|? ]
@@ -659,6 +698,40 @@ =head2 dSource arguments
 =item B<-hooks path_to_hooks>
 Import hooks exported using dx_get_hooks
 
+=item B<-customparameters (param_name=value)|(#param_name)>
+Provide a custom parameter for Postgresql. For more then one parameter, use -customparameters multiple times.
+To comment an existing variable inside Postgresql VDB, put a # sign before a parameter.
+
+=item B<-ingestiontype single|initalized|externalbackup >
+Postgresql dSource ingestion type
+
+=item B<-sourcehostname hostname>
+Postgresql dSource source host
+
+=item B<-sourceport xxx>
+Postgresql dSource source port
+
+=item B<-singledbname dbname>
+Postgresql dSource single database name
+
+=item B<-stagingport xxx>
+Postgresql dSource staging port
+
+=item B<-dumpdir directory>
+Postgresql dSource single database dump directory
+
+=item B<-restorejobs x>
+Postgresql dSource single database number of restore jobs
+
+=item B<-dumpjobs x>
+Postgresql dSource single database number of dump joba
+
+=item B<-backup_dir_log>
+Location of WAL logs for external Postgresql ingestion
+
+=item B<-keepinsync yes|no->
+Keep a dSource in sync using Postgresql replication. If value set to yes, replication parameters are mandatory
+
 =back
 
 =head2 Hooks
@@ -808,6 +881,27 @@ =head1 EXAMPLES
  Waiting for all actions to complete. Parent action is ACTION-2919
  Action completed with success
 
+Adding a Posgresql dSource using Delphix initialized backup
+
+ dx_ctl_dsource -d dxtest -action create -group "Untitled" -creategroup -dsourcename "postdsource"  -type postgresql -sourcename "postdsource" -stageinst "Postgres vFiles (15.2)" \
+                -stageenv "POSTSTG" -stage_os_user "postgres" -mountbase "/mnt/provision/postdsource" -password xxxxxxxxx -stagingport 5433 -ingestiontype initiated \
+                -dbuser "delphix" -sourcehostname "sourceserver" -sourceport 5432 -customparameters "deadlock_timeout=123s"
+ Waiting for all actions to complete. Parent action is ACTION-734
+ Action completed with success
+
+Adding a Posgresql dSource using single database ingestion
+
+dx_ctl_dsource -d dxtest -action create -group "Untitled" -creategroup -dsourcename "postdsource"  -type postgresql -sourcename "postdsource_cluster" -stageinst "Postgres vFiles (15.2)" \
+               -stageenv "POSTSTG" -mountbase "/mnt/provision/pioro" -dbuser "postgres" -password xxxxxxxxx -stagingport 5433 -sourcehostname "sourceserver" \
+               -ingestiontype single  -sourceport 5432 -singledbname "singleDB" -dumpdir "/home/postgres" -restorejobs 2 -dumpjobs 2
+
+Adding a Postgresql dSource using external backup
+
+ dx_ctl_dsource -d dxtest -action create -group "Untitled" -creategroup -dsourcename "extbac" -type postgresql -sourcename "extbac" -stageinst "Postgres vFiles (15.2)" \
+                -stageenv "POSTSTG"  -mountbase "/mnt/provision/extbac" -stagingport 5434 -ingestiontype externalbackup \
+                -backup_dir "/home/postgres/backup" -backup_dir_log "/home/postgres/backup" -keepinsync no
+
+
 Updating a backup path and validated sync mode for Sybase
 
  dx_ctl_dsource -d Landshark5 -action update -validatedsync ENABLED -backup_dir "/u02/sybase_back" -dsourcename pubs3
@@ -839,4 +933,6 @@ =head1 EXAMPLES
   Action completed with success
 
 
+
+
 =cut
diff --git a/bin/dx_ctl_env.pl b/bin/dx_ctl_env.pl
@@ -410,7 +410,7 @@
         } elsif (lc $repotype eq 'postgresql') {
           my %native_params;
           my %plugin_params_hash = (
-            "prettyName" => $dbname
+            "name" => $dbname
           );
           if ($sourceconfig_obj->createSourceConfig('plugin', $repo->{reference}, $dbname, \%native_params, \%plugin_params_hash)) {
             print "Can't add Postgresql $dbname \n";

diff --git a/bin/dx_get_dsourcesize.pl b/bin/dx_get_dsourcesize.pl
@@ -170,7 +170,7 @@
 
       if ($dbobj->getEnvironmentName() eq 'NA') {
         # staging push
-        $envname = $dbobj->getStagingEnvironment();
+        $envname = $dbobj->getStagingEnvironmentName();
       } else {
         $envname = $dbobj->getEnvironmentName();
       }

diff --git a/bin/dx_provision_vdb.pl b/bin/dx_provision_vdb.pl
@@ -51,6 +51,7 @@
 my $version = $Toolkit_helpers::version;
 
 my $archivelog = 'yes';
+my $datapatch = 'no';
 
 GetOptions(
   'help|?' => \(my $help),
@@ -122,6 +123,8 @@
   'tdeexportsecret=s' => \(my $tdeexportsecret),
   'tdecdbpassword=s' => \(my $tdecdbpassword),
   'tdekeyid=s' => \(my $tdekeyid),
+  'customparameters=s@' => \(my $customparameters),
+  'datapatch=s' => \($datapatch),
   'dever=s' => \(my $dever),
   'debug:n' => \(my $debug),
   'all' => (\my $all),
@@ -171,7 +174,7 @@
 }
 
 
-if ( ! ( ( $type eq 'oracle') || ( $type eq 'mssql') || ( $type eq 'sybase') || ( $type eq 'mysql') ||( $type eq 'db2') || ( $type eq 'vFiles') ) )  {
+if ( ! ( ( $type eq 'oracle') || ( $type eq 'mssql') || ( $type eq 'sybase') || ( $type eq 'mysql') ||( $type eq 'db2') || ( $type eq 'vFiles') || ( $type eq 'postgresql') ) )  {
   print "Option -type has invalid parameter - $type \n";
   pod2usage(-verbose => 1, -input=>\*DATA);
   exit (1);
@@ -288,6 +291,8 @@
     $db = new DB2VDB_obj($engine_obj,$debug);
   } elsif ($type eq 'vFiles') {
     $db = new AppDataVDB_obj($engine_obj,$debug);
+  } elsif ($type eq 'postgresql') {
+    $db = new PostgresVDB_obj($engine_obj,$debug);
   }
 
   # common database code
@@ -554,7 +559,15 @@
       $db->setNewDBID();
     }
 
-
+    if (defined($datapatch)) {
+      if ((lc $datapatch eq 'no') || ( lc $datapatch eq 'yes') ) {
+        $db->setDataPatch($datapatch);
+      } else {
+        print "Datapatch argument should be yes or no. VDB won't be created\n" ;
+        $ret = $ret + 1;
+        next;
+      }
+    }
 
     if ( defined($template) ) {
       if ( $db->setTemplate($template) ) {
@@ -688,6 +701,16 @@
     }
     $db->setName($targetname, $dbname);
     $jobno = $db->createVDB($group,$environment,$envinst);
+  } elsif ($type eq 'postgresql') {
+    if (! defined($port)) {
+        print "Port not defined. VDB won't be created.\n";
+        $ret = $ret + 1;
+        next;
+    }
+
+    $db->setName($targetname, $dbname);    
+    $jobno = $db->createVDB($group,$environment,$envinst,$mntpoint,$port, $customparameters);
+
   }
 
   if (defined($snapshotpolicy_ref)) {
@@ -778,13 +801,14 @@ =head1 SYNOPSIS
                    -tdeexportsecret tde_export_secret]
                   [-tdekeyid tde_key_id]
                   [-tdecdbpassword tde_cdb_keystore_password]
+                  [-customparameters (param_name=value)|(#param_name)]
                   [-help] [-debug]
 
 
 =head1 DESCRIPTION
 
 Provision VDB from a defined source on the defined target environment.
-
+dx_provision_vdb.pl -d dxtest -type postgresql -group "Untitled" -creategroup -sourcename "pioro"  -srcgroup "Untitled" -targetname "postvdb"  -dbname "postvdb" -environment "marcinposttgt.dlpxdc.co"  -envinst "Postgres vFiles (15.2)"  -envUser "postgres"  -hooks /tmp/postvdb.dbhooks  -mntpoint "/mnt/provision/postvdb" -port "5444" -customparameters "deadlock_timeout=123s" -customparameters "#max_connections"
 =head1 ARGUMENTS
 
 =head2 Delphix Engine selection - if not specified a default host(s) from dxtools.conf will be used.
@@ -1006,6 +1030,9 @@ =head2 VDB arguments
 =item B<-tdecdbpassword tde_cdb_keystore_password>
 Password for an existing target CDB keystore (this is an optional parameter is password of the CDB is already set)
 
+=item B<-customparameters (param_name=value)|(#param_name)>
+Provide a custom parameter for Postgresql. For more then one parameter, use -customparameters multiple times.
+To comment an existing variable inside Postgresql VDB, put a # sign before a parameter.
 
 =back
 
@@ -1159,4 +1186,11 @@ =head1 EXAMPLES
  Job JOB-158167 finised with state: COMPLETED VDB created.
 
 
+Provision a Postgresql VDB 
+
+ dx_provision_vdb -d dxtest -type postgresql -group "Untitled" -sourcename "pioro" -targetname "postvdb"  -dbname "postvdb" -environment "POSTTGT"  -envinst "Postgres vFiles (15.2)"  -mntpoint "/mnt/provision/postvdb" -port "5444" -customparameters "deadlock_timeout=123s" -customparameters "#max_connections"
+ Starting job - JOB-608
+ 0 - 7 - 11 - 18 - 75 - 100
+ Job JOB-608 finished with state: COMPLETED
+ VDB created.
 =cut
diff --git a/lib/Bookmark_obj.pm b/lib/Bookmark_obj.pm
@@ -217,6 +217,8 @@ sub createBookmark
         $bookmark_timeflow_type = 'AppDataTimeflowPoint'
     } elsif ($db->getDBType() eq 'vFiles') {
         $bookmark_timeflow_type = 'AppDataTimeflowPoint'
+    } elsif ($db->getDBType() eq 'postgresql') {
+        $bookmark_timeflow_type = 'AppDataTimeflowPoint'
     } else {
       print "Can't determine a DB type. Exiting\n";
       return 1;

diff --git a/lib/Databases.pm b/lib/Databases.pm
@@ -41,6 +41,8 @@ use MSSQLVDB_obj;
 use SybaseVDB_obj;
 use AppDataVDB_obj;
 use DB2VDB_obj;
+use PluginVDB_obj;
+use PostgresVDB_obj;
 use Toolkit_obj;
 use Toolkit_helpers qw (logger);
 use Encode qw(decode_utf8);
@@ -136,10 +138,6 @@ sub LoadDBList
         {
             $db = MSSQLVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
         }
-        elsif ($dbitem->{type} eq 'PgSQLDatabaseContainer' )
-        {
-            $db = PostgresVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
-        }
         elsif ($dbitem->{type} eq 'ASEDBContainer' )
         {
             $db = SybaseVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
@@ -153,6 +151,8 @@ sub LoadDBList
             if (defined($dbitem->{toolkit}) && defined($toolkits->getName($dbitem->{toolkit})) ) {
                 if ($toolkits->getName($dbitem->{toolkit}) eq 'db2db') {
                   $db = DB2VDB_obj->new($self->{_dlpxObject}, $self->{_debug});
+                } elsif ($toolkits->getName($dbitem->{toolkit}) eq 'postgres-vsdk') {
+                  $db = PostgresVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
                 } else {
                   $db = AppDataVDB_obj->new($self->{_dlpxObject}, $self->{_debug});
                 }

diff --git a/lib/MSSQLVDB_obj.pm b/lib/MSSQLVDB_obj.pm
@@ -116,7 +116,7 @@ sub getConfig
       $config = join($joinsep,($config, "-recoveryModel $recoveryModel"));
     } elsif ($self->getType() eq 'dSource')  {
       my $staging_user = $self->getStagingUser();
-      my $staging_env = $self->getStagingEnvironment();
+      my $staging_env = $self->getStagingEnvironmentName();
       my $staging_inst = $self->getStagingInst();
 
       $config = join($joinsep,($config, "-stageinst \"$staging_inst\""));