Merge branch 'master' into production

gen/atlassian_mu

@@ -10,25 +10,36 @@ use utf8;
 
 local $::SERVICE_NAME = "atlassian_mu";
 local $::PROTOCOL_VERSION = "3.0.0";
-my $SCRIPT_VERSION = "3.0.0";
+my $SCRIPT_VERSION = "3.0.1";
 
 perunServicesInit::init;
 my $DIRECTORY = perunServicesInit::getDirectory;
 my $data = perunServicesInit::getHashedHierarchicalData;
 
-#forward declaration
+# Forward declaration
 sub processUsers;
 sub processResource;
 sub processMemberships;
 
-#Constants
+# Constants
 our $A_USER_ID;                        *A_USER_ID =                        \'urn:perun:user:attribute-def:core:id';
 our $A_USER_LOGIN;                     *A_USER_LOGIN =                     \'urn:perun:user_facility:attribute-def:virt:login';
 our $A_USER_GIVEN_NAME;                *A_USER_GIVEN_NAME =                \'urn:perun:user:attribute-def:core:firstName';
 our $A_USER_FAMILY_NAME;               *A_USER_FAMILY_NAME =               \'urn:perun:user:attribute-def:core:lastName';
 our $A_RESOURCE_ATLASSIAN_GROUP_NAME;  *A_RESOURCE_ATLASSIAN_GROUP_NAME =  \'urn:perun:resource:attribute-def:def:atlassianGroupName';
+our $A_FACILITY_ATLASSIAN_DOMAIN;      *A_FACILITY_ATLASSIAN_DOMAIN =      \'urn:perun:facility:attribute-def:def:atlassianDomain';
+our $A_MEMBER_STATUS;                  *A_MEMBER_STATUS =                  \'urn:perun:member:attribute-def:core:status';
 
-my $domain = "\@muni.cz";
+our $STATUS_VALID;                     *STATUS_VALID =                     \'VALID';
+
+# Hard limits might depend on directory! (Reference limits: https://support.atlassian.com/provisioning-users/docs/understand-user-provisioning/)
+my $MEMBERS_HARD_LIMIT = 35000;
+my $USERS_HARD_LIMIT = 150000;
+
+if (!defined($data->getFacilityAttributeValue( attrName => $A_FACILITY_ATLASSIAN_DOMAIN ))) {
+	exit 1;
+}
+my $domain = $data->getFacilityAttributeValue( attrName => $A_FACILITY_ATLASSIAN_DOMAIN );
 
 my $userStruc = {};
 my $resourceStruc = {};
@@ -46,6 +57,11 @@ foreach my $resourceId ($data->getResourceIds()) {
 
 # PREPARE USERSDATA TO JSON
 my %users = ();
+my $usersSize = keys %$userStruc;
+if ($usersSize >= $USERS_HARD_LIMIT) {
+	print STDERR "Users limit (" . $USERS_HARD_LIMIT . " users) reached!\n";
+	exit 1;
+}
 foreach my $uid (sort keys %$userStruc) {
 	my $userInfo = {};
 	$userInfo->{$emailHeader} = $userStruc->{$uid}->{$emailHeader};
@@ -68,6 +84,11 @@ foreach my $resourceId (sort keys %$resourceStruc) {
 		push @members, $userStruc->{$uid}->{$emailHeader};
 	}
 
+	if (scalar(@members) >= $MEMBERS_HARD_LIMIT) {
+		print STDERR "Members limit (" . $MEMBERS_HARD_LIMIT . " members) reached on resource #" . $resourceId  . "!\n";
+		exit 1;
+	}
+
 	$groups{$resourceStruc->{$resourceId}->{$A_RESOURCE_ATLASSIAN_GROUP_NAME}} = \@members;
 }
 
@@ -86,13 +107,17 @@ perunServicesInit::finalize;
 sub processUsers {
 	my ($resourceId, $memberId) = @_;
 
+	if ($data->getMemberAttributeValue( member => $memberId, attrName => $A_MEMBER_STATUS ) ne $STATUS_VALID) {
+		return;
+	}
+
 	my $uid = $data->getUserAttributeValue( member => $memberId, attrName => $A_USER_ID );
 	my $userFamilyName = $data->getUserAttributeValue( member => $memberId, attrName => $A_USER_FAMILY_NAME );
 	my $userGivenName = $data->getUserAttributeValue( member => $memberId, attrName => $A_USER_GIVEN_NAME );
 	my $login = $data->getUserFacilityAttributeValue( member => $memberId, facility => $data->getFacilityId, attrName => $A_USER_LOGIN );
 
 	unless (exists $userStruc->{$uid}) {
-		$userStruc->{$uid}->{$emailHeader} = $login . $domain;
+		$userStruc->{$uid}->{$emailHeader} = $login . '@' . $domain;
 		$userStruc->{$uid}->{$familyNameHeader} = $userFamilyName;
 		$userStruc->{$uid}->{$givenNameHeader} = $userGivenName;
 	}

gen/pithia_portal

@@ -0,0 +1,45 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use perunServicesInit;
+use perunServicesUtils;
+use JSON::XS;
+use utf8;
+
+local $::SERVICE_NAME = "pithia_portal";
+local $::PROTOCOL_VERSION = "1.0.0";
+my $SCRIPT_VERSION = "1.0.0";
+
+perunServicesInit::init;
+my $DIRECTORY = perunServicesInit::getDirectory;
+my $data = perunServicesInit::getHashedDataWithGroups;
+
+#Constants
+our $A_GROUP_NAME;               *A_GROUP_NAME =        \'urn:perun:group:attribute-def:core:name';
+
+my @groupNames = ();
+
+foreach my $resourceId ($data->getResourceIds()) {
+	foreach my $groupId ($data->getGroupIdsForResource(resource => $resourceId)) {
+		my $groupName = $data->getGroupAttributeValue(group => $groupId, attrName => $A_GROUP_NAME);
+
+		# skip top-level hierarchy group assigned to their single facility
+		# skip all sub-groups of each organization since registrations are only for the organizations.
+		if ($groupName eq "organizations" || $groupName =~ m/^organizations:[^:]+:(.)+/ ) {
+			next;
+		}
+		push(@groupNames, $groupName);
+	}
+}
+
+@groupNames = sort(@groupNames);
+
+# PRINT DATA TO JSON FILE
+my $file = "$DIRECTORY/$::SERVICE_NAME.json";
+open FILE,">$file" or die "Cannot open $file: $! \n";
+binmode(FILE);
+print FILE JSON::XS->new->utf8->pretty->canonical->encode(\@groupNames);
+close (FILE) or die "Cannot close $file: $! \n";
+
+perunServicesInit::finalize;

send/atlassian_mu_process.pl

@@ -20,7 +20,7 @@
 my $DEBUG=0;
 
 # Required setting for connection
-my $RESULTS_COUNT = 200; # This many objects will be fetched with one GET request on server
+my $RESULTS_COUNT = 200; # This many objects will be fetched with one GET request on server. Limited by Atlassian ANYWAY!
 my $directoryUrl;
 my $key;
 
@@ -37,6 +37,7 @@
 my @ATLASSIAN_ERROR_CODES = (500, 503); # Internal Atlassian error codes
 my $WAITING_ERROR_TIME = 20 * 1000; # Wait this long (ms) if above error occurs
 my $REPEAT_ERROR_COUNT = 3; # Retry this many times if above error occurs
+my $GROUP_OPERATIONS_LIMIT = 9999; # Limit of group operations in a chunk on Atlassian side
 
 my $USERS_FILENAME = 'users.scim';
 my $GROUPS_FILENAME = 'groups.scim';
@@ -192,7 +193,7 @@ sub fetchTheirUsers {
 	my %theirUsersHash = ();
 
 	while ($startIndex <= $totalResults) {
-		my $url = $directoryUrl . 'Users?startIndex=' . $startIndex . '&count=' . $RESULTS_COUNT;
+		my $url = $directoryUrl . '/Users?startIndex=' . $startIndex . '&count=' . $RESULTS_COUNT;
 		my $response = callServer($url, $TYPE_GET);
 		$totalResults = $response->{'totalResults'};
 		$itemsPerPage = $response->{'itemsPerPage'};
@@ -217,7 +218,7 @@ sub fetchTheirGroups {
 	my %theirGroupsHash = ();
 
 	while ($startIndex <= $totalResults) {
-		my $url = $directoryUrl . 'Groups?startIndex=' . $startIndex . '&count=' . $RESULTS_COUNT;
+		my $url = $directoryUrl . '/Groups?startIndex=' . $startIndex . '&count=' . $RESULTS_COUNT;
 		my $response = callServer($url, $TYPE_GET);
 		$totalResults = $response->{'totalResults'};
 		$itemsPerPage = $response->{'itemsPerPage'};
@@ -290,11 +291,20 @@ sub fetchOurGroups {
 # Compares Atlassian users with Perun users.
 # Sends request to create missing users, update outdated user data and deactivate users missing in Perun.
 sub evaluateUsers {
+	my $theirUserName;
+	my $theirUserInfo;
+	while (($theirUserName, $theirUserInfo) = each(%{$theirUsers})) {
+		my $ourUserInfo = $ourUsers->{$theirUserName};
+		if (!$ourUserInfo && $theirUserInfo->{'active'}) {
+			updateUser($theirUserName, $theirUserInfo, $theirUserInfo->{'id'}, 0);
+		}
+	}
+
 	my $ourUserName;
 	my $ourUserInfo;
 	while (($ourUserName, $ourUserInfo) = each(%{$ourUsers})) {
-		my $theirUserInfo = $theirUsers->{$ourUserName};
-		if ($theirUserInfo) {
+		$theirUserInfo = $theirUsers->{$ourUserName};
+		if (defined $theirUserInfo) {
 			# check if attributes changed or our user got active again
 			if (checkUserAttributesDiffer($ourUserInfo, $theirUserInfo) || !$theirUserInfo->{"active"}) {
 				updateUser($ourUserName, $ourUserInfo, $theirUserInfo->{'id'}, 1);
@@ -303,15 +313,6 @@ sub evaluateUsers {
 			createUser($ourUserName, $ourUserInfo);
 		}
 	}
-
-	my $theirUserName;
-	my $theirUserInfo;
-	while (($theirUserName, $theirUserInfo) = each(%{$theirUsers})) {
-		my $ourUserInfo = $ourUsers->{$theirUserName};
-		if (!$ourUserInfo && $theirUserInfo->{'active'}) {
-			updateUser($theirUserName, $theirUserInfo, $theirUserInfo->{'id'}, 0);
-		}
-	}
 }
 
 # Compares Atlassian groups with Perun groups.
@@ -357,15 +358,15 @@ sub createGroup {
 
 	my $data = { "displayName" => $displayName };
 	$groupsCreated++;
-	return callServer($directoryUrl . 'Groups', $TYPE_POST, JSON::XS->new->utf8->encode($data));
+	return callServer($directoryUrl . '/Groups', $TYPE_POST, JSON::XS->new->utf8->encode($data));
 }
 
 # Sends request to remove a group from Atlassian.
 sub removeGroup {
 	my $groupId = shift;
 	if ($DEBUG) {print "Removing group $groupId.\n";}
 
-	callServer($directoryUrl . 'Groups/' . $groupId, $TYPE_DELETE);
+	callServer($directoryUrl . '/Groups/' . $groupId, $TYPE_DELETE);
 	$groupsRemoved++;
 }
 
@@ -395,19 +396,37 @@ sub updateGroupMembership {
 		push @membersToRemove, {"value" => $theirUsers->{$outdatedMember}->{"id"}, "display" => $outdatedMember};
 	}
 
-	my $updateContent = {
-		"schemas"    => [ "urn:ietf:params:scim:api:messages:2.0:PatchOp" ],
-		"Operations" => [
-			{"op" => "add",
-			"path" => "members",
-			"value" => \@membersToAdd},
-			{"op" => "remove",
-			"path" => "members",
-			"value" => \@membersToRemove}
-		]
-	};
+	# Chunks need to be used for membership operations because of operations limit in Atlassian
+	my @chunks_to_remove = ();
+	push @chunks_to_remove, [ splice @membersToRemove, 0, $GROUP_OPERATIONS_LIMIT ] while @membersToRemove;
+	foreach (@chunks_to_remove) {
+		my $updateContent = {
+			"schemas"    => [ "urn:ietf:params:scim:api:messages:2.0:PatchOp" ],
+			"Operations" => [
+				{ "op"      => "remove",
+					"path"  => "members",
+					"value" => \@$_}
+			]
+		};
+
+		callServer($directoryUrl . '/Groups/' . $groupId, $TYPE_PATCH, JSON::XS->new->utf8->encode($updateContent));
+	}
+
+	my @chunks_to_add = ();
+	push @chunks_to_add, [ splice @membersToAdd, 0, $GROUP_OPERATIONS_LIMIT ] while @membersToAdd;
+	foreach (@chunks_to_add) {
+		my $updateContent = {
+			"schemas"    => [ "urn:ietf:params:scim:api:messages:2.0:PatchOp" ],
+			"Operations" => [
+				{"op" => "add",
+					"path" => "members",
+					"value" => \@$_}
+			]
+		};
+
+		callServer($directoryUrl . '/Groups/' . $groupId, $TYPE_PATCH, JSON::XS->new->utf8->encode($updateContent));
+	}
 
-	callServer($directoryUrl . 'Groups/' . $groupId, $TYPE_PATCH, JSON::XS->new->utf8->encode($updateContent));
 	$membershipUpdated++;
 }
 
@@ -430,7 +449,7 @@ sub updateUser {
 		"active"   => $statusActive ? JSON::XS::true : JSON::XS::false
 	};
 
-	callServer($directoryUrl . 'Users/' . $userId, $TYPE_PUT, JSON::XS->new->utf8->encode($updateContent));
+	callServer($directoryUrl . '/Users/' . $userId, $TYPE_PUT, JSON::XS->new->utf8->encode($updateContent));
 	if ($statusActive) { $usersUpdated++ } else { $usersDeactivated++ };
 }
 

send/pithia_portal

@@ -0,0 +1,4 @@
+#!/bin/bash
+export SERVICE_NAME="pithia_portal"
+
+python3 generic_send.py "$1" "$2" "$3"

send/send_lib.py

@@ -5,6 +5,7 @@
 import os
 import tempfile
 import shutil
+from typing import Optional
 
 PERUN_CERT = "/etc/perun/ssl/perun-send.pem"
 PERUN_KEY = "/etc/perun/ssl/perun-send.key"
@@ -291,7 +292,7 @@ def get_windows_proxy() -> str:
 	return windows_proxy
 
 
-def load_custom_transport_command(service_name: str) -> list[str] | None:
+def load_custom_transport_command(service_name: str) -> Optional[list[str]]:
 	"""
 	Retrieves custom transport command from config file located in service directory.
 	Config file must end with .py -> "/etc/perun/services/service_name/service_name.py")