Merge branch 'development' into epos-msl

.github/workflows/api-and-integration-tests.yml

@@ -5,6 +5,7 @@ on:
     branches:
     - development
     - release-1.9
+    - release-1.10
     - "**-atr"
     # We can force an integration/API test run without opening a PR by pushing to a branch name that ends with "-atr"
   pull_request:
@@ -38,6 +39,8 @@ jobs:
       run: |
         if [ "${{ steps.extract_branch.outputs.branch }}" = "release-1.9" ]; then
           echo "branch=release-1.9" >> $GITHUB_OUTPUT
+        elif [ "${{ steps.extract_branch.outputs.branch }}" = "release-1.10" ]; then
+          echo "branch=release-1.10" >> $GITHUB_OUTPUT
         else
           echo "branch=development" >> $GITHUB_OUTPUT
         fi
@@ -111,7 +114,7 @@ jobs:
         cd tests
         nohup bash -c 'while true ; do sleep 5 ;  ../yoda/docker/run-cronjob.sh copytovault >> ../copytovault.log 2>&1 ; ../yoda/docker/run-cronjob.sh publication >> ../publication.log 2>&1 ; done' &
         test -d mycache || mkdir -p mycache
-        python3 -m pytest --skip-ui --intake --datarequest --deposit -o cache_dir=mycache --environment environments/docker.json
+        python3 -m pytest --skip-ui --datarequest --deposit -o cache_dir=mycache --environment environments/docker.json
         cat ../copytovault.log
         cat ../publication.log
 

.github/workflows/api-documentation.yml

@@ -55,13 +55,11 @@ jobs:
           export PYTHONPATH="${PYTHONPATH}:."
           python tools/api/generate-openapi.py rules_uu --module datarequest > build/api_datarequest.json
           python tools/api/generate-openapi.py rules_uu --module deposit > build/api_deposit.json
-          python tools/api/generate-openapi.py rules_uu --module intake > build/api_intake.json
 
       - name: Validate Yoda module API documentation
         run: |
           openapi-spec-validator build/api_datarequest.json
           openapi-spec-validator build/api_deposit.json
-          openapi-spec-validator build/api_intake.json
 
       - name: Deploy 🚀
         uses: JamesIves/github-pages-deploy-action@releases/v3

.github/workflows/build-push-image.yml

@@ -6,6 +6,7 @@ on:
     branches:
       - 'development'
       - 'release-1.9'
+      - 'release-1.10'
 
 jobs:
   push-image:

__init__.py

@@ -26,39 +26,36 @@
 # Import all modules containing rules into the package namespace,
 # so that they become visible to iRODS.
 
-from admin                  import *
-from browse                 import *
-from folder                 import *
-from groups                 import *
-from json_datacite          import *
-from json_landing_page      import *
-from mail                   import *
-from meta                   import *
-from meta_form              import *
-from provenance             import *
-from research               import *
-from resources              import *
-from schema                 import *
-from schema_transformation  import *
-from schema_transformations import *
-from vault                  import *
-from datacite               import *
-from epic                   import *
-from publication            import *
-from policies               import *
-from replication            import *
-from revisions              import *
-from settings               import *
-from notifications          import *
-from integration_tests      import *
+from admin                    import *
+from browse                   import *
+from folder                   import *
+from groups                   import *
+from json_datacite            import *
+from json_landing_page        import *
+from mail                     import *
+from meta                     import *
+from meta_form                import *
+from provenance               import *
+from research                 import *
+from resources                import *
+from schema                   import *
+from schema_transformation    import *
+from schema_transformations   import *
+from publication_troubleshoot import *
+from vault                    import *
+from datacite                 import *
+from epic                     import *
+from publication              import *
+from policies                 import *
+from replication              import *
+from revisions                import *
+from settings                 import *
+from notifications            import *
+from integration_tests        import *
 
 # Import certain modules only when enabled.
 from .util.config import config
 
-if config.enable_intake:
-    from intake import *
-    from intake_vault import *
-
 if config.enable_datarequest:
     from datarequest import *
 

folder.py

@@ -205,7 +205,7 @@ def precheck_folder_secure(ctx, coll):
     found, last_run = get_last_run_time(ctx, coll)
     if (not correct_copytovault_start_status(ctx, coll)
             or not correct_copytovault_start_location(coll)
-            or not misc.last_run_time_acceptable(coll, found, last_run, config.vault_copy_backoff_time)):
+            or not misc.last_run_time_acceptable(found, last_run, config.vault_copy_backoff_time)):
         return False
 
     return True

groups.py

@@ -25,6 +25,7 @@
            'rule_group_remove_external_user',
            'rule_group_check_external_user',
            'rule_group_expiration_date_validate',
+           'rule_user_exists',
            'rule_group_user_exists',
            'api_group_search_users',
            'api_group_exists',
@@ -56,18 +57,13 @@ def getGroupsData(ctx):
         attr = row[1]
         value = row[2]
 
-        # Create/update group with this information.
-        try:
-            group = groups[name]
-        except Exception:
-            group = {
-                "name": name,
-                "managers": [],
-                "members": [],
-                "read": [],
-                "invited": []
-            }
-            groups[name] = group
+        group = groups.setdefault(name, {
+            "name": name,
+            "managers": [],
+            "members": [],
+            "read": [],
+            "invited": []
+        })
 
         if attr in ["schema_id", "data_classification", "category", "subcategory"]:
             group[attr] = value
@@ -95,26 +91,17 @@ def getGroupsData(ctx):
             if name.startswith("read-"):
                 # Match read-* group with research-* or initial-* group.
                 name = name[5:]
-                try:
-                    # Attempt to add to read list of research group.
-                    group = groups["research-" + name]
-                    group["read"].append(user)
-                except Exception:
-                    try:
-                        # Attempt to add to read list of initial group.
-                        group = groups["initial-" + name]
+                for prefix in ("research-", "initial-"):
+                    group = groups.get(prefix + name)
+                    if group:
                         group["read"].append(user)
-                    except Exception:
-                        pass
+                        break
             elif not name.startswith("vault-"):
-                try:
-                    # Ordinary group.
-                    group = groups[name]
+                group = groups.get(name)
+                if group:
                     group["members"].append(user)
-                except KeyError:
-                    pass
 
-    # Third query: obtain list of invited SRAM users
+    # Third query: obtain list of invited SRAM users.
     if config.enable_sram:
         iter = genquery.row_iterator(
             "META_USER_ATTR_VALUE, USER_NAME, USER_ZONE",
@@ -124,11 +111,9 @@ def getGroupsData(ctx):
         for row in iter:
             name = row[0]
             user = row[1] + "#" + row[2]
-            try:
-                group = groups[name]
+            group = groups.get(name)
+            if group:
                 group["invited"].append(user)
-            except KeyError:
-                pass
 
     return groups.values()
 
@@ -529,11 +514,11 @@ def api_group_process_csv(ctx, csv_header_and_data, allow_update, delete_users):
         return api.Error('errors', validation_errors)
 
     # Step 3: Create / update groups.
-    error = apply_data(ctx, data, allow_update, delete_users)
-    if len(error):
-        return api.Error('errors', [error])
+    status_msg = apply_data(ctx, data, allow_update, delete_users)
+    if status_msg['status'] == 'error':
+        return api.Error('errors', [status_msg['message']])
 
-    return api.Result.ok()
+    return api.Result.ok(info=[status_msg['message']])
 
 
 def validate_data(ctx, data, allow_update):
@@ -553,7 +538,7 @@ def validate_data(ctx, data, allow_update):
     for (category, subcategory, groupname, _managers, _members, _viewers, _schema_id, _expiration_date) in data:
 
         if group.exists(ctx, groupname) and not allow_update:
-            errors.append('Group "{}" already exists'.format(groupname))
+            errors.append('Group "{}" already exists. It has not been updated.'.format(groupname))
 
         # Is user admin or has category add privileges?
         if not (is_admin or can_add_category):
@@ -575,11 +560,13 @@ def apply_data(ctx, data, allow_update, delete_users):
     :param allow_update: Allow updates in groups
     :param delete_users:  Allow for deleting of users from groups
 
-    :returns: Errors if found any
+    :returns: Errors if found any, or message with actions if everything is successful
     """
 
     for (category, subcategory, group_name, managers, members, viewers, schema_id, expiration_date) in data:
         new_group = False
+        users_added, users_removed = 0, 0
+        message = ''
 
         log.write(ctx, 'CSV import - Adding and updating group: {}'.format(group_name))
 
@@ -590,10 +577,12 @@ def apply_data(ctx, data, allow_update, delete_users):
 
         if response:
             new_group = True
+            message += "Group '{}' created.".format(group_name)
         elif response.status == "error_group_exists" and allow_update:
             log.write(ctx, 'CSV import - WARNING: group "{}" not created, it already exists'.format(group_name))
+            message += "Group '{}' already exists.".format(group_name)
         else:
-            return "Error while attempting to create group {}. Status/message: {} / {}".format(group_name, response.status, response.status_info)
+            return {status: 'error', message: "Error while attempting to create group {}. Status/message: {} / {}".format(group_name, response.status, response.status_info)}
 
         # Now add the users and set their role if other than member
         allusers = managers + members + viewers
@@ -604,6 +593,7 @@ def apply_data(ctx, data, allow_update, delete_users):
                 if response:
                     currentrole = "normal"
                     log.write(ctx, "CSV import - Notice: added user {} to group {}".format(username, group_name))
+                    users_added += 1
                 else:
                     log.write(ctx, "CSV import - Warning: error occurred while attempting to add user {} to group {}".format(username, group_name))
                     log.write(ctx, "CSV import - Status: {} , Message: {}".format(response.status, response.status_info))
@@ -669,11 +659,21 @@ def apply_data(ctx, data, allow_update, delete_users):
                     response = group_remove_user_from_group(ctx, username, usergroupname)
                     if response:
                         log.write(ctx, "CSV import - Removing user {} from group {}".format(username, usergroupname))
+                        users_removed += 1
                     else:
                         log.write(ctx, "CSV import - Warning: error while attempting to remove user {} from group {}".format(username, usergroupname))
                         log.write(ctx, "CSV import - Status: {} , Message: {}".format(response.status, response.status_info))
 
-    return ''
+        if users_added > 0:
+            message += ' Users added ({}).'.format(users_added)
+        if users_removed > 0:
+            message += ' Users removed ({}).'.format(users_removed)
+
+        # If no users added, no users removed and not new group created.
+        if not users_added and not users_removed and not new_group:
+            message += ' No changes made.'
+
+    return {"status": "ok", "message": message}
 
 
 def _are_roles_equivalent(a, b):
@@ -705,6 +705,18 @@ def group_user_exists(ctx, group_name, username, include_readonly):
         return False
 
 
+@rule.make(inputs=[0], outputs=[1])
+def rule_user_exists(ctx, username):
+    """Rule wrapper to check if a user exists.
+
+    :param ctx:      Combined type of a ctx and rei struct
+    :param username: User to check for existence
+
+    :returns: Indicator if user exists
+    """
+    return "true" if user.exists(ctx, username) else "false"
+
+
 def rule_group_user_exists(rule_args, callback, rei):
     """Check if a user is a member of the given group.
 
@@ -973,12 +985,15 @@ def group_create(ctx, group_name, category, subcategory, schema_id, expiration_d
             if not sram.sram_connect_service_collaboration(ctx, short_name):
                 return api.Error('sram_error', 'Something went wrong connecting service to group "{}" in SRAM'.format(group_name))
 
+        if group.exists(ctx, group_name):
+            return api.Error('group_exists', "Group {} not created, it already exists".format(group_name))
+
         response = ctx.uuGroupAdd(group_name, category, subcategory, schema_id, expiration_date, description, data_classification, co_identifier, '', '')['arguments']
         status = response[8]
         message = response[9]
         if status == '0':
             return api.Result.ok()
-        elif status == '-1089000' or status == '-809000':
+        elif status == '-1089000' or status == '-809000' or status == '-806000':
             return api.Error('group_exists', "Group {} not created, it already exists".format(group_name))
         else:
             return api.Error('policy_error', message)

groups_import.py

@@ -142,7 +142,7 @@ def parse_csv_file(ctx):
     # Start processing the actual group data rows
     for line in lines:
         row_number += 1
-        rowdata, error = process_csv_line(line)
+        rowdata, error = process_csv_line(ctx, line)
 
         if error is None:
             extracted_data.append(rowdata)