Final push.

Author: LTLA

README.md

@@ -339,9 +339,9 @@ Administrators should refresh these statistics manually as described above after
 ### Deleting content (admin)
 
 Administrators have the ability to delete files from the registry.
-This violates **gypsum**'s immutability contract and should be done sparingly.
-In particular, administrators must ensure that no other project links to the to-be-deleted files, otherwise those links will be invalidated.
-This check involves going through all the manifest files and is currently a manual process.
+This violates the Gobbler's immutability contract and should be done sparingly.
+In particular, administrators must ensure that no other project links to the to-be-deleted files, otherwise those links will be invalidated -
+see the ["Rerouting symlinks"](#rerouting-symlinks-admin) section for details.
 
 To delete a project, create a file with the `request-delete_project-` prefix.
 This file should be JSON-formatted with the following properties:
@@ -379,6 +379,35 @@ On success, the version is deleted.
 The HTTP response will contain a JSON object with the `type` property set to `SUCCESS`.
 A success is still reported even if the version, its asset or its project is not present, in which case the operation is a no-op.
 
+### Rerouting symlinks (admin)
+
+In the (hopefully rare) scenario where one or more directories must be deleted from the registry,
+administrators must consider the possibility that other projects in the registry contain symbolic links to the files in the to-be-deleted directories.
+Deletion would result in dangling links that compromise the validity of those other projects. 
+To avoid this, the Gobbler can reroute each affected link to a more appropriate location,
+either by updating the link target or replacing it with a copy of the to-be-deleted file.
+After successful rerouting, each project, asset or version can be safely deleted without damaging other projects.
+
+To reroute links, create a file with the `request-reroute_links-` prefix.
+This file should be JSON-formatted with the following properties:
+
+- `to_delete`: an array of JSON objects.
+  Each object corresponds to a project, asset or version directory to be deleted.
+  For a project directory, the object should contain a `project` string property that names the project;
+  for an asset directory, the object should contain the `project` and `asset` string properties;
+  and for a version directory, the object should contain the `project`, `asset` and `version` string properties.
+
+On success, the Gobbler will update any links in the registry to any file in the directories corresponding to `delete`. 
+All internal metadata files (`..manifest`, `..links`) are similarly updated to mirror the changes on the filesystem.
+The HTTP response will contain a JSON object with the `status` property set to `SUCCESS`.
+
+Note that a rerouting request does not actually delete the directories corresponding to `to_delete`.
+After rerouting, administrators still need to delete each project, asset or version [as described above](#deleting-content-admin).
+If an administrator is sure that there are no links targeting a directory, deletion can be performed directly without the expense of rerouting. 
+
+We use a `to_delete` array to batch together multiple deletion tasks.
+This improves efficiency by amortizing the cost of a full registry scan to find links that target any of the affected directories.
+
 ## Parsing logs
 
 For some actions, the Gobbler creates a log within the `..logs/` subdirectory of the registry.

main.go

@@ -144,6 +144,8 @@ func main() {
             reportable_err = deleteAssetHandler(reqpath, &globals)
         } else if strings.HasPrefix(reqtype, "delete_version-") {
             reportable_err = deleteVersionHandler(reqpath, &globals)
+        } else if strings.HasPrefix(reqtype, "reroute_links-") {
+            reportable_err = rerouteLinksHandler(reqpath, &globals)
         } else if strings.HasPrefix(reqtype, "reindex_version-") {
             reportable_err = reindexHandler(reqpath, &globals)
         } else if strings.HasPrefix(reqtype, "health_check-") { // TO-BE-DEPRECATED, see /check below.

reroute.go

@@ -234,7 +234,9 @@ func rerouteLinksHandler(reqpath string, globals *globalConfiguration) error {
     }
 
     // First we validate the request.
-    all_incoming := []deleteTask{}
+    all_incoming := struct {
+        ToDelete []deleteTask `json:"to_delete"`
+    }{}
     contents, err := os.ReadFile(reqpath)
     if err != nil {
         return fmt.Errorf("failed to read %q; %w", reqpath, err)
@@ -243,9 +245,11 @@ func rerouteLinksHandler(reqpath string, globals *globalConfiguration) error {
     err = json.Unmarshal(contents, &all_incoming)
     if err != nil {
         return newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err))
+    } else if all_incoming.ToDelete == nil {
+        return newHttpError(http.StatusBadRequest, fmt.Errorf("expected a 'to_delete' property in %q; %w", reqpath, err))
     }
 
-    for _, incoming := range all_incoming {
+    for _, incoming := range all_incoming.ToDelete {
         err := isMissingOrBadName(&(incoming.Project))
         if err != nil {
             return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err))
@@ -269,7 +273,7 @@ func rerouteLinksHandler(reqpath string, globals *globalConfiguration) error {
     }
 
     // Then we need to reroute the links.
-    to_delete_versions, err := listToBeDeletedVersions(globals.Registry, all_incoming)
+    to_delete_versions, err := listToBeDeletedVersions(globals.Registry, all_incoming.ToDelete)
     if err != nil {
         return err
     }

reroute_test.go

@@ -762,7 +762,11 @@ func TestRerouteLinksHandler(t *testing.T) {
             t.Fatal(err)
         }
 
-        reqpath, err := dumpRequest("reroute_links", fmt.Sprintf(`[ { "project": "%s", "asset": "%s", "version": "origination" } ]`, project, asset))
+        reqpath, err := dumpRequest("reroute_links", fmt.Sprintf(`{
+    "to_delete": [ 
+        { "project": "%s", "asset": "%s", "version": "origination" } 
+    ]
+}`, project, asset))
         if err != nil {
             t.Fatalf("failed to dump a request type; %v", err)
         }
@@ -826,10 +830,12 @@ func TestRerouteLinksHandler(t *testing.T) {
             t.Fatal(err)
         }
 
-        reqpath, err := dumpRequest("reroute_links", fmt.Sprintf(`[ 
-    { "project": "%s", "asset": "%s", "version": "animation" },
-    { "project": "%s", "asset": "%s", "version": "natural" }
-]`, project, asset, project, asset))
+        reqpath, err := dumpRequest("reroute_links", fmt.Sprintf(`{
+    "to_delete": [ 
+        { "project": "%s", "asset": "%s", "version": "animation" },
+        { "project": "%s", "asset": "%s", "version": "natural" }
+    ]
+}`, project, asset, project, asset))
         if err != nil {
             t.Fatalf("failed to dump a request type; %v", err)
         }
@@ -899,7 +905,7 @@ func TestRerouteLinksHandler(t *testing.T) {
             t.Fatal(err)
         }
 
-        reqpath, err := dumpRequest("reroute_links", `[ { "project": "ARIA" } ]`)
+        reqpath, err := dumpRequest("reroute_links", `{ "to_delete": [ { "project": "ARIA" } ] }`)
         if err != nil {
             t.Fatalf("failed to dump a request type; %v", err)
         }
@@ -924,25 +930,34 @@ func TestRerouteLinksHandler(t *testing.T) {
             t.Fatal(err)
         }
 
-        reqpath, err := dumpRequest("reroute_links", `[ { "project": "" } ]`)
+        globals := newGlobalConfiguration(registry)
+        globals.Administrators = append(globals.Administrators, self)
+
+        reqpath, err := dumpRequest("reroute_links", "{}")
         if err != nil {
             t.Fatalf("failed to dump a request type; %v", err)
         }
+        err = rerouteLinksHandler(reqpath, &globals)
+        if err == nil || !strings.Contains(err.Error(), "'to_delete'") {
+            t.Error("expected failure when to_delete isn't present")
+        }
 
-        globals := newGlobalConfiguration(registry)
-        globals.Administrators = append(globals.Administrators, self)
+        reqpath, err = dumpRequest("reroute_links", `{ "to_delete": [ { "project": "" } ] }`)
+        if err != nil {
+            t.Fatalf("failed to dump a request type; %v", err)
+        }
         err = rerouteLinksHandler(reqpath, &globals)
         if err == nil || !strings.Contains(err.Error(), "invalid 'project'") {
             t.Error("expected failure from invalid project")
         }
 
-        reqpath, err = dumpRequest("reroute_links", `[ { "project": "ARIA", "asset": "" } ]`)
+        reqpath, err = dumpRequest("reroute_links", `{ "to_delete": [ { "project": "ARIA", "asset": "" } ] }`)
         err = rerouteLinksHandler(reqpath, &globals)
         if err == nil || !strings.Contains(err.Error(), "invalid 'asset'") {
             t.Error("expected failure from invalid asset")
         }
 
-        reqpath, err = dumpRequest("reroute_links", `[ { "project": "ARIA", "asset": "anime", "version": "" } ]`)
+        reqpath, err = dumpRequest("reroute_links", `{ "to_delete": [ { "project": "ARIA", "asset": "anime", "version": "" } ] }`)
         err = rerouteLinksHandler(reqpath, &globals)
         if err == nil || !strings.Contains(err.Error(), "invalid 'version'") {
             t.Error("expected failure from invalid version")