From 4018e04f86af393c2219d40f7e86838e26c7310d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johana=20Sup=C3=ADkov=C3=A1?= Date: Wed, 19 Apr 2023 10:42:20 +0200 Subject: [PATCH 1/4] feat(dhus): dhus service * dhus service sends data to keycloak and database * slave script calls script which will be set on the destination machine * all attributes that are set on service are process by gen script --- gen/dhus | 12 ++++++ gen/perunDataGenerator.pm | 9 ++++ send/dhus | 4 ++ slave/process-dhus/bin/process-dhus.sh | 41 +++++++++++++++++++ slave/process-dhus/changelog | 5 +++ .../conf/example-pre_10_set_variables | 7 ++++ slave/process-dhus/dependencies | 1 + slave/process-dhus/rpm.dependencies | 1 + slave/process-dhus/short_desc | 1 + 9 files changed, 81 insertions(+) create mode 100755 gen/dhus create mode 100755 send/dhus create mode 100755 slave/process-dhus/bin/process-dhus.sh create mode 100644 slave/process-dhus/changelog create mode 100755 slave/process-dhus/conf/example-pre_10_set_variables create mode 100644 slave/process-dhus/dependencies create mode 100644 slave/process-dhus/rpm.dependencies create mode 100644 slave/process-dhus/short_desc diff --git a/gen/dhus b/gen/dhus new file mode 100755 index 00000000..6697d131 --- /dev/null +++ b/gen/dhus @@ -0,0 +1,12 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use File::Basename; +use perunDataGenerator; + +local $::SERVICE_NAME = basename($0); +local $::PROTOCOL_VERSION = "3.0.0"; +local $::SKIP_NON_VALID_MEMBERS = 1; + +perunDataGenerator::generateUsersDataInJSON; \ No newline at end of file diff --git a/gen/perunDataGenerator.pm b/gen/perunDataGenerator.pm index 8a6a351c..6c1bc5a0 100644 --- a/gen/perunDataGenerator.pm +++ b/gen/perunDataGenerator.pm @@ -9,11 +9,15 @@ use Exporter 'import'; our $JSON_FORMAT = "json"; our @EXPORT = qw($JSON_FORMAT); +our $A_MEMBER_STATUS; *A_MEMBER_STATUS = \'urn:perun:member:attribute-def:core:status'; + # Generate user and user_facility required attributes for each user into JSON file. # Subroutine uses perunServicesInit which REQUIRE access to $::SERVICE_NAME and $::PROTOCOL_VERSION. # This can be achieved by following lines in your main script: (for example) # local $::SERVICE_NAME = "passwd"; # local $::PROTOCOL_VERSION = "3.0.0"; +# If not valid VO members should be skipped, member status attribute needs to be set on service and set +# local $::SKIP_NON_VALID_MEMBERS = 1; sub generateUsersDataInJSON { perunServicesInit::init; @@ -45,6 +49,11 @@ sub generateUsersDataInJSON { ####### prepare data ###################### my %usersIds = (); foreach my $memberId ($data->getMemberIdsForFacility()) { + + if ($::SKIP_NON_VALID_MEMBERS) { + next if $data->getMemberAttributeValue( member => $memberId, attrName => $A_MEMBER_STATUS ) ne 'VALID'; + } + my $userId = $data->getUserIdForMember(member => $memberId); if (exists($usersIds{$userId})) { next; diff --git a/send/dhus b/send/dhus new file mode 100755 index 00000000..10066b44 --- /dev/null +++ b/send/dhus @@ -0,0 +1,4 @@ +#!/bin/bash +export SERVICE_NAME="dhus" + +python3 generic_sender.py "$1" "$2" "$3" \ No newline at end of file diff --git a/slave/process-dhus/bin/process-dhus.sh b/slave/process-dhus/bin/process-dhus.sh new file mode 100755 index 00000000..db2db9cb --- /dev/null +++ b/slave/process-dhus/bin/process-dhus.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +PROTOCOL_VERSION='3.0.0' + +function process { + + E_MISSING_DST_PATH=(50 'Missing path of handling script (DST_SCRIPT), need to be set in pre_script.') + E_MISSING_DST_EXIST=(51 'Handling script does not exist at the specified location (' + "${DST_SCRIPT}" + '), please check that the correct path is set in pre_script') + E_MISSING_DST_EXEC=(52 'Handling script is not executable (' + "${DST_SCRIPT}" + '), please check that the correct permissions are set') + + E_MISSING_CONF_PATH=(53 'Missing path of configuration (DST_CONF), need to be set in pre_script.') + E_MISSING_CONF_EXIST=(54 'Configuration does not exist at the specified location (' + "${DST_CONF}" + '), please check that the correct path is set in pre_script') + + if [ -z ${DST_SCRIPT} ]; then + log_msg E_MISSING_DST_PATH + fi + + if [ ! -f ${DST_SCRIPT} ]; then + log_msg E_MISSING_DST_EXIST + fi + + if [ ! -x ${DST_SCRIPT} ]; then + log_msg E_MISSING_DST_EXEC + fi + + if [ -z ${DST_CONF} ]; then + log_msg E_MISSING_CONF_PATH + fi + + if [ ! -d ${DST_CONF} ]; then + log_msg E_MISSING_CONF_EXIST + fi + + create_lock + + FROM_PERUN="${WORK_DIR}" + + ${DST_SCRIPT} -f $FROM_PERUN -c $DST_CONF + + exit $? +} diff --git a/slave/process-dhus/changelog b/slave/process-dhus/changelog new file mode 100644 index 00000000..2af727e8 --- /dev/null +++ b/slave/process-dhus/changelog @@ -0,0 +1,5 @@ +perun-slave-process-dhus (3.0.0) stable; urgency=low + + * New service dhus + + -- Johana Supikova Wed, 12 Apr 2023 14:32:00 +0200 diff --git a/slave/process-dhus/conf/example-pre_10_set_variables b/slave/process-dhus/conf/example-pre_10_set_variables new file mode 100755 index 00000000..0db1d810 --- /dev/null +++ b/slave/process-dhus/conf/example-pre_10_set_variables @@ -0,0 +1,7 @@ +#!/bin/sh + +# set handling script +#DST_SCRIPT=/tmp/dhus.py + +# set configuration path - specific config will be retrieved by facility name (e.g. /etc/dhus/facility.yml) +#DST_CONF=/etc/dhus/ diff --git a/slave/process-dhus/dependencies b/slave/process-dhus/dependencies new file mode 100644 index 00000000..a6717984 --- /dev/null +++ b/slave/process-dhus/dependencies @@ -0,0 +1 @@ +perun-slave-base diff --git a/slave/process-dhus/rpm.dependencies b/slave/process-dhus/rpm.dependencies new file mode 100644 index 00000000..a6717984 --- /dev/null +++ b/slave/process-dhus/rpm.dependencies @@ -0,0 +1 @@ +perun-slave-base diff --git a/slave/process-dhus/short_desc b/slave/process-dhus/short_desc new file mode 100644 index 00000000..90719b2e --- /dev/null +++ b/slave/process-dhus/short_desc @@ -0,0 +1 @@ +Package for perun service - dhus From 05cd694d15c3c474402ed4141b052f97e6930f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zl=C3=A1mal?= Date: Mon, 24 Apr 2023 12:36:43 +0200 Subject: [PATCH 2/4] refactor(k5login_root): removed usage of listOfDestinations attribute This feature was not used at all. Service is now simplified with single output on all destinations. --- gen/k5login_root | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/gen/k5login_root b/gen/k5login_root index 587e0d49..331db356 100755 --- a/gen/k5login_root +++ b/gen/k5login_root @@ -8,47 +8,36 @@ use File::Basename; local $::SERVICE_NAME = basename($0); local $::PROTOCOL_VERSION = "3.0.0"; -my $SCRIPT_VERSION = "3.1.2"; +my $SCRIPT_VERSION = "3.2.0"; perunServicesInit::init; +my $directory = perunServicesInit::getDirectory; my $data = perunServicesInit::getHashedDataWithGroups; - our $A_PRINCIPAL; *A_PRINCIPAL = \'urn:perun:user:attribute-def:def:kerberosAdminPrincipal'; our $A_USER_STATUS; *A_USER_STATUS = \'urn:perun:member:attribute-def:core:status'; -our $A_GROUP_DESTINATIONS; *A_GROUP_DESTINATIONS = \'urn:perun:group:attribute-def:def:listOfDestinations'; -my %outputByDestination = ('all' => undef); +my $kerberosLogins = (); # $kerberosLogins->{principal} = 1 foreach my $resourceId ($data->getResourceIds()) { - foreach my $groupId ($data->getGroupIdsForResource( resource => $resourceId )) { - my @generatedDataDestination = ('all'); - if(defined $data->getGroupAttributeValue(group => $groupId, attrName => $A_GROUP_DESTINATIONS )) { - @generatedDataDestination = @{$data->getGroupAttributeValue(group => $groupId, attrName => $A_GROUP_DESTINATIONS )}; - } + foreach my $memberId ($data->getMemberIdsForResource(resource => $resourceId)) { - foreach my $memberId ($data->getMemberIdsForResourceAndGroup(resource => $resourceId, group => $groupId )) { - next if $data->getMemberAttributeValue( member => $memberId, attrName => $A_USER_STATUS ) ne 'VALID'; - push @{$outputByDestination{$_}}, $data->getUserAttributeValue( member => $memberId, attrName => $A_PRINCIPAL ) foreach @generatedDataDestination; - } + next if $data->getMemberAttributeValue( member => $memberId, attrName => $A_USER_STATUS ) ne 'VALID'; - } -} + my $principal = $data->getUserAttributeValue( member => $memberId, attrName => $A_PRINCIPAL ); + $kerberosLogins->{$principal} = 1; + } -####### output ###################### -{ - local $, = "\n"; - local $\ = "\n"; +} - for my $destination (keys %outputByDestination) { - my $destinationDirectory = getDestinationDirectory $destination; - my $service_file_name = "$destinationDirectory/$::SERVICE_NAME"; - open SERVICE_FILE,">$service_file_name" or die "Cannot open $service_file_name: $! \n"; - print SERVICE_FILE sort &uniqList(@{$outputByDestination{$destination}}, @{$outputByDestination{"all"}}); - close(SERVICE_FILE); - } +####### output file ###################### +my $service_file_name = "$directory/$::SERVICE_NAME"; +open SERVICE_FILE,">$service_file_name" or die "Cannot open $service_file_name: $! \n"; +foreach my $principal (sort keys %$kerberosLogins) { + print SERVICE_FILE $principal . "\n"; } +close(SERVICE_FILE); perunServicesInit::finalize; From 7122fb97606e5cefa51bbd480023466416990b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johana=20Sup=C3=ADkov=C3=A1?= Date: Thu, 16 Mar 2023 13:44:00 +0100 Subject: [PATCH 3/4] feat(tinia): use python send script * rewrote tinia from bash to python as another example service * used send_lib library --- send/send_lib.py | 2 +- send/tinia | 138 +++++++++-------------------------------------- 2 files changed, 27 insertions(+), 113 deletions(-) diff --git a/send/send_lib.py b/send/send_lib.py index b147fd2c..7ea0cc4a 100644 --- a/send/send_lib.py +++ b/send/send_lib.py @@ -150,7 +150,7 @@ def prepare_temporary_directory() -> tempfile.TemporaryDirectory: so it is removed afterwards with all its content. :return: created temporary directory """ - return tempfile.TemporaryDirectory(prefix="perun-send.", dir=TEMPORARY_DIR, ignore_cleanup_errors=True) + return tempfile.TemporaryDirectory(prefix="perun-send.", dir=TEMPORARY_DIR) def copy_files_to_directory(path_from: str, path_to: str, name_pattern: re.Pattern = None) -> None: diff --git a/send/tinia b/send/tinia index 5f6e1cdc..d6ed7f73 100755 --- a/send/tinia +++ b/send/tinia @@ -1,119 +1,33 @@ -#!/bin/bash +#!/usr/bin/env python3 -SERVICE_NAME="tinia" +import send_lib +import sys +import re -FACILITY_NAME=$1 -DESTINATION=$2 -DESTINATION_TYPE=$3 +service_name = "tinia" -if [ -z "$DESTINATION" ]; then - echo "Missing Destination argument (DB NAME there)" >&2 - exit 231 -fi +send_lib.check_input_fields(sys.argv, destination_type_required=True) -if [ -z "$FACILITY_NAME" ]; then - echo "Missing FacilityName argument" >&2 - exit 232 -fi +facility = sys.argv[1] +destination = sys.argv[2] +destination_type = sys.argv[3] -if [ -z "$DESTINATION_TYPE" ]; then - echo "Destination type of this service can't be empty" >&2 - exit 233; -else - TYPE="service-specific" - if [ "$DESTINATION_TYPE" != "$TYPE" ]; then - echo "Destination type of this service need to be $TYPE" >&2 - exit 234; - fi -fi +send_lib.check_destination_type_allowed(destination_type, "service-specific") +send_lib.check_destination_format(destination, destination_type, send_lib.SIMPLE_PATTERN) #Destination is name of database on oracle in localhost tnsnames file -DBNAME=$DESTINATION - -SERVICE_FILES_BASE_DIR="`pwd`/../gen/spool" -SERVICE_FILES_DIR="$SERVICE_FILES_BASE_DIR/$FACILITY_NAME/$SERVICE_NAME" - -#Just safety check. This should not happen. -if [ ! -d "$SERVICE_FILES_DIR" ]; then echo '$SERVICE_FILES_DIR: '$SERVICE_FILES_DIR' is not a directory' >&2 ; exit 1; fi - -#Create lock (same like in slave) -LOCK_DIR=${LOCK_DIR:=/var/lock} -LOCK_FILE="${LOCK_DIR}/perunv3-${SERVICE_NAME}-$DBNAME.lock" -LOCK_PIDFILE="$LOCK_FILE/pid" - -function create_lock { - if mkdir "${LOCK_FILE}"; then - trap 'rm -r -f "${LOCK_FILE}"' EXIT - echo $$ > "$LOCK_PIDFILE"; - if [ $? -ne 0 ]; then - echo "Can't create lock file." >&2 - exit 250 - fi - else - # lock file exists, check for existence of concurrent process - if ps ax | grep "$SERVICE_NAME" | sed 's/^\([0-9]\+\).*/\1/' | grep "\(^\| \)`cat $LOCK_PIDFILE`\( \|$\)"; then - # concurrent process is running - this skript must terminate - echo "Concuret process tinia_process is running" >&2 - exit 249 - else - # lock is not valid; it should be deleted - rm -r "$LOCK_FILE" - if [ $? -ne 0 ]; then - echo "Can't remove not valid lock file." >&2 - exit 248 - fi - echo "Invalid lock file found and deleted: $LOCK_FILE" >&2 - mkdir "${LOCK_FILE}" - if [ $? -ne 0 ]; then - echo "Can't create lock after removing invalid lock." >&2 - exit 247 - fi - trap 'rm -r -f "${LOCK_FILE}"' EXIT - echo $$ > "$LOCK_PIDFILE" - if [ $? -ne 0 ]; then - echo "Can't create lock file after removing invalid lock file." >&2 - exit 246 - fi - fi - fi -} - -create_lock - -TMP_HOSTNAME_DIR="`mktemp -d /tmp/perun-send.XXXXXXXXXX`" -if [ $? -ne 0 ]; then - echo "Can't create temporary dir" >&2 - exit 255 -fi - -#prepare removing of temporary files and dirs after exit of script -trap 'rm -r -f "${LOCK_FILE}" "${TMP_HOSTNAME_DIR}"' EXIT - -cp $SERVICE_FILES_DIR/$SERVICE_NAME $TMP_HOSTNAME_DIR -if [ $? -ne 0 ]; then - echo "Can't copy service file to temporary dir" >&2 - exit 254 -fi - -EXECSCRIPT="./tinia_process.pl" - -if [ ! -f "$EXECSCRIPT" ]; then - echo "Can't locate process script!" >&2 - exit 253 -fi - -$EXECSCRIPT -d $DBNAME -p $TMP_HOSTNAME_DIR -s $SERVICE_NAME - -ERRORCODE=$? -if [ $ERRORCODE -ne 0 ]; then - echo "Process exit with error" >&2 - exit $ERRORCODE -fi - -ERR_CODE=$? - -if [ $ERR_CODE -ne 0 ]; then - echo "Slave script ends with return code: $ERR_CODE" >&2 -fi - -exit $ERR_CODE +dbname = destination + +service_files_dir = send_lib.get_gen_folder(facility, service_name) +send_lib.create_lock(service_name, dbname) + +# copy service file to temporary directory +with send_lib.prepare_temporary_directory() as tmp_dir: + send_lib.copy_files_to_directory(service_files_dir, tmp_dir, re.compile(service_name)) + scriptpath = "./tinia_process.pl" + process = send_lib.exec_script(scriptpath, ["-d", dbname, "-p", tmp_dir, "-s", service_name]) + process.wait() + stdout, stderr = process.communicate() + print(stdout.decode()) + if process.returncode != 0: + send_lib.die_with_error("Slave script ends with return code: " + str(process.returncode)) From ca2b937eb7e7f1b129dad2df823d6988748374a6 Mon Sep 17 00:00:00 2001 From: Michal Berky Date: Thu, 4 May 2023 16:33:20 +0200 Subject: [PATCH 4/4] fix(zabbix_mu): fix script to be able to work with null value of role attribute Removed unnecessary if statement that caused issues when role was null. Changed the slave script to be able to work with any script and not be limited to jar files. --- gen/zabbix_mu | 3 --- slave/process-zabbix-mu/bin/process-zabbix_mu.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/gen/zabbix_mu b/gen/zabbix_mu index 3cc3a104..3468e342 100755 --- a/gen/zabbix_mu +++ b/gen/zabbix_mu @@ -40,9 +40,6 @@ foreach my $resourceId ($data->getResourceIds()) { unless ($userData->{$uco} || (!defined $groupName && !defined $role)) { my $firstName = $data->getUserAttributeValue( member => $memberId, attrName => $A_USER_FIRSTNAME ); my $lastName = $data->getUserAttributeValue( member => $memberId, attrName => $A_USER_LASTNAME ); - if (!defined $role) { - $role = []; - } my $user = { identifier => $uco, firstName => $firstName, diff --git a/slave/process-zabbix-mu/bin/process-zabbix_mu.sh b/slave/process-zabbix-mu/bin/process-zabbix_mu.sh index 02b7f58d..e2b7f0de 100755 --- a/slave/process-zabbix-mu/bin/process-zabbix_mu.sh +++ b/slave/process-zabbix-mu/bin/process-zabbix_mu.sh @@ -24,7 +24,7 @@ function process { FROM_PERUN="${WORK_DIR}/zabbix_mu.json" - java -jar ${DST_SCRIPT} $FROM_PERUN + ${DST_SCRIPT} $FROM_PERUN exit $? }