Merge pull request #60 from OS2borgerPC/47-versionering-af-globalecor…

…e-scripts

47 versionering af globalecore scripts

admin_site/requirements.txt

@@ -21,3 +21,5 @@ psycopg==3.1.18                  # Used in the example docker-compose.yml
 python-dateutil==2.8.2           # Required by django-xmlrpc
 requests==2.31.0
 whitenoise==6.6.0                # If you don't have a web server in front to serve static files
+PyYAML==6.0.2
+markdown==3.7

admin_site/system/forms.py

@@ -120,15 +120,32 @@ class Meta:
         model = PCGroup
         exclude = ["site", "configuration", "wake_week_plan"]
 
+class NewScriptForm(forms.ModelForm):
+    def __init__(self, *args, **kwargs):
+        super(NewScriptForm, self).__init__(*args, **kwargs)
+        instance = getattr(self, "instance", None)
+        if instance and instance.pk:
+            self.fields["site"].disabled = True
+
+        self.fields["tags"].disabled = True
+
+    class Meta:
+        model = Script
+        exclude = ["feature_permission", "product"]
 
 class ScriptForm(forms.ModelForm):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, global_script = False, **kwargs):
         super(ScriptForm, self).__init__(*args, **kwargs)
         instance = getattr(self, "instance", None)
         if instance and instance.pk:
             self.fields["site"].disabled = True
 
         self.fields["tags"].disabled = True
+        self.fields['name'].widget.attrs['disabled'] = global_script
+        self.fields['name'].required = not global_script
+        self.fields['description'].widget.attrs['disabled'] = global_script
+        self.fields['description'].required = not global_script
+
 
     class Meta:
         model = Script

admin_site/system/management/commands/fetch_and_install_core_scripts.py

@@ -0,0 +1,62 @@
+import django
+from django.core.management.base import BaseCommand
+from system.models import Script, ScriptTag, Input
+from system.script_fetcher import fetch_scripts
+
+class Command(BaseCommand):
+
+    """
+    Get core scripts from a (correctly formatted) git repo, and insert them in the database, so they can be accessed by site users.
+    For security and consistency reasons, the matching commit hash (SHA-256) for the tag must also be specified, as the tag can easily be updated to point to 
+    other (ill-intended) code. Using the hash, this is prevented.
+
+    Example:
+        manage.py fetch_and_install_core_scripts --versionTag v0.1.2 --commitHash b3a791b52bc9937c6cb168c706ee003b0666fc93
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument("--versionTag", required=True)
+        parser.add_argument("--commitHash", required=True)
+
+    def handle(self, *args, **options):
+        repo_url = "https://github.com/OS2borgerPC/os2borgerpc-core-scripts.git"
+        self.stdout.write(f"Fetching scripts from {repo_url} repository...")
+        scripts = fetch_scripts("https://github.com/OS2borgerPC/os2borgerpc-core-scripts.git", options['versionTag'], options['commitHash'])
+
+        for script in scripts:
+            versionedName = script.title + " " + options['versionTag']
+            uid = script.metadata.get("uid", None)
+            is_security_script = script.metadata.get("security", False)
+            is_hidden = script.metadata.get("hidden", False)
+
+            if uid and Script.objects.filter(uid=uid).exists():
+                Script.objects.filter(uid=uid).delete()
+
+            if not Script.objects.filter(name=versionedName).exists():
+                with open(script.sourcePath, 'rb') as file:
+                    # Get only the base file name
+                    db_script = Script.objects.create(
+                        name=versionedName,
+                        description=script.description,
+                        site=None, # None means global script
+                        executable_code=django.core.files.File(file),
+                        is_security_script=is_security_script,
+                        is_hidden=is_hidden,
+                        maintained_by_magenta=False,
+                        feature_permission=None,
+                        uid=uid
+                    )
+                    tag, created = ScriptTag.objects.get_or_create(name=script.tag)
+                    db_script.tags.add(tag)
+
+                    position = 1
+                    for parameter in script.parameters:
+                        Input.objects.create(
+                            script=db_script,
+                            name=parameter.name,
+                            value_type=parameter.type,
+                            default_value=parameter.default,
+                            position=position,
+                            mandatory=parameter.mandatory
+                        )
+                        position += 1

admin_site/system/migrations/0087_alter_script_executable_code.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-11-05 15:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('system', '0086_remove_featurepermission_sites_remove_site_country_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='script',
+            name='executable_code',
+            field=models.FileField(max_length=255, upload_to='script_uploads', verbose_name='executable code'),
+        ),
+    ]

admin_site/system/models.py

@@ -858,7 +858,7 @@ class Script(AuditModelMixin):
     # script OR a single extractable .zip or .tar.gz file with all necessary
     # data.
     executable_code = models.FileField(
-        verbose_name=_("executable code"), upload_to="script_uploads"
+        verbose_name=_("executable code"), upload_to="script_uploads", max_length=255
     )
     is_security_script = models.BooleanField(
         verbose_name=_("security script"), default=False, null=False

admin_site/system/script_fetcher.py

@@ -0,0 +1,146 @@
+import os
+import subprocess
+from pathlib import Path
+from urllib.parse import urlparse
+import yaml
+from dataclasses import dataclass, field
+from typing import List, Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+def fetch_scripts(repoUrl, versionTag, commitHash):
+    repo_url = repoUrl
+    clone_path = Path("downloaded_core_scripts") / get_repo_name(repo_url) / commitHash
+
+    # Perform a shallow clone if the repository isn't already cloned
+    if not (clone_path).exists():
+        if versionTag:
+            # Check that the commitHash matches the versionTag
+            subprocess.run(
+                ["git", "clone", "--depth", "1", "--branch", versionTag, repo_url, str(clone_path)], 
+                check=True
+            )
+
+            versionTagCommitHash = subprocess.run(
+                ["git", "-C", str(clone_path), "rev-parse", versionTag], 
+                check=True, 
+                capture_output=True,
+                text=True
+            ).stdout.strip()
+
+            if versionTagCommitHash != commitHash:
+                logger.warning(f"The commit hash for tag '{versionTag}' is '{versionTagCommitHash}', which does not match the provided commit hash '{commitHash}'.")
+                return []
+        else:
+            # Clone the repository and then check out the specific commit hash
+            subprocess.run(
+                ["git", "clone", "--filter=blob:none", repo_url, str(clone_path)], 
+                check=True
+            )
+            subprocess.run(
+                ["git", "-C", str(clone_path), "checkout", commitHash],
+                check=True
+            )
+
+    # Retrieve content of all markdown (.md) files, and convert them to Scripts
+    scripts = []
+    md_dir_path = clone_path
+    for md_file in md_dir_path.glob("**/*.md"):
+        if md_file.is_file():
+            if md_file.name == "README.md":
+                continue
+            with open(md_file, 'r') as file:
+                file_content = file.read()
+                try:
+                    script = parse_md_to_script(file_content, clone_path)
+                except Exception as e:
+                    logger.warning("Skipping script file '" + str(md_file.relative_to(md_dir_path)) + "' because it's content is not formatted correctly:")
+                    logger.warning(file_content)
+                    logger.warning(f"Exception: {e}")
+                    continue
+                if not Path(script.sourcePath).exists() and not Path(script.sourcePath).is_file():
+                    logger.warning("Skipping " + str(md_file.relative_to(md_dir_path)) + " because the 'source' attribute does not point to a valid file.")
+                    continue
+                scripts.append(script)
+
+    return scripts
+
+@dataclass
+class Metadata:
+    uid: Optional[str]
+    security: Optional[bool]
+    hidden: Optional[bool]
+
+@dataclass
+class Parameter:
+    name: str
+    type: str
+    default: Optional[str]
+    mandatory: bool
+
+@dataclass
+class Script:
+    title: str
+    parent: str
+    sourcePath: str
+    compatible_versions: Optional[str]
+    compatible_images: List[str]
+    description: str
+    tag: str
+    partners: Optional[str]
+    parameters: List[Parameter] = field(default_factory=list)
+    metadata: Optional[Metadata] = None
+
+def parse_md_to_script(content: str, clone_path: Path) -> Script:
+    # Split YAML and Markdown content
+    yamlBegin, yaml_string, markdown_string = content.split("---", 2)
+    yaml_string = yaml_string.strip()
+    markdown_string = markdown_string.strip()
+
+    # Parse the YAML content
+    yaml_content = yaml.safe_load(yaml_string)
+
+    # Parse metadata
+    metadata_content = yaml_content.get('metadata', {})
+
+    # Parse parameters
+    params = yaml_content.get('parameters', [])
+    params = params if params is not None else []
+
+    # Extract parameters to list
+    parameters = [
+        Parameter(
+            name=param['name'],
+            type=param['type'].upper(),
+            default=param['default'],
+            mandatory=param['mandatory']
+        )
+        for param in params
+    ]
+
+    return Script(
+        title=yaml_content.get('title'),
+        parent=yaml_content.get('parent'),
+        sourcePath=str(clone_path / yaml_content.get('source')),
+        compatible_versions=yaml_content.get('compatible_versions'),
+        compatible_images=yaml_content.get('compatible_images', []),
+        description=markdown_string,
+        tag=yaml_content.get('parent'),
+        partners=yaml_content.get('partners'),
+        parameters=parameters,
+        metadata=metadata_content
+    )
+
+def get_repo_name(repo_url):
+    # Parse the URL
+    parsed_url = urlparse(repo_url)
+
+    # Extract the path from the URL and split to get the repository name
+    repo_name = os.path.basename(parsed_url.path)
+
+    # Remove the ".git" extension if present
+    if repo_name.endswith(".git"):
+        repo_name = repo_name[:-4]
+
+    return repo_name

admin_site/system/templatetags/markdown_extras.py

@@ -0,0 +1,9 @@
+import markdown
+from django import template
+from django.utils.safestring import mark_safe
+
+register = template.Library()
+
+@register.filter(name='markdown')
+def markdown_format(text):
+    return mark_safe(markdown.markdown(text))

admin_site/system/views.py

@@ -81,6 +81,7 @@
     PCGroupForm,
     ParameterForm,
     ScriptForm,
+    NewScriptForm,
     SecurityEventForm,
     SiteForm,
     SiteCreateForm,
@@ -1330,6 +1331,20 @@ class ScriptUpdate(ScriptMixin, UpdateView, SuperAdminOrThisSiteMixin):
     template_name = "system/scripts/update.html"
     form_class = ScriptForm
 
+    # This get_form method is overriden to pass global_script to the form_class (ScripForm) instance
+    def get_form(self, form_class=None):
+        if form_class is None:
+            form_class = self.get_form_class()
+
+        script_instance = self.get_object()
+        global_script = script_instance.is_global if script_instance else False
+
+        # Ensure the original arguments are passed to the form, along with global_script
+        form_kwargs = self.get_form_kwargs()
+        form_kwargs["global_script"] = global_script
+
+        return form_class(**form_kwargs)
+
     def get_context_data(self, **kwargs):
         # Get context from super class
         context = super(ScriptUpdate, self).get_context_data(**kwargs)
@@ -1342,7 +1357,7 @@ def get_context_data(self, **kwargs):
                 display_code = "<Kan ikke vise koden - upload venligst igen.>"
             context["script_preview"] = display_code
         context["type_choices"] = Input.VALUE_CHOICES
-        self.create_form = ScriptForm()
+        self.create_form = NewScriptForm()
         self.create_form.prefix = "create"
         context["create_form"] = self.create_form
         context["is_hidden"] = self.script.is_hidden

admin_site/templates/system/scripts/NewScriptForm.html

@@ -0,0 +1,40 @@
+{% load crispy_forms_tags %}
+
+{% load i18n %}
+<fieldset>
+  {{ form.name|as_crispy_field }}
+  {{ form.description|as_crispy_field }}
+
+    <input
+      type="hidden"
+      id="id_{% if form.prefix %}{{ form.prefix }}-{% endif %}site"
+      name="{% if form.prefix %}{{ form.prefix }}-{% endif %}site"
+      value="{{ site.id }}">
+    <input
+      id="id_is_security_script"
+      name="is_security_script"
+      value="{{ is_security }}"
+      type="hidden">
+    <input
+      id="id_is_hidden"
+      name="is_hidden"
+      value="{{ is_hidden }}"
+      type="hidden">
+    <input
+      id="id_uid"
+      name="uid"
+      value="{{ uid }}"
+      type="hidden">
+</fieldset>
+
+      <fieldset {% if disable_inputs == "yes" %}class="mt-3"{% endif %}>
+        {% if not global_script %}
+          {{ form.executable_code|as_crispy_field }}
+        {% endif %}
+        {% if show_code_preview %}
+          <p>{% translate "Code" %}:</p>
+          <pre class="mb-0"><code id="script-code" class="bash">{{script_preview |escape}}</code></pre>
+        {% endif %}
+
+      </fieldset>
+      <input type="hidden" name="script-number-of-inputs" class="script-number-of-inputs" value="0">