Load dependencies from .modules subfolder

CHANGELOG.md

@@ -22,13 +22,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - #702 Preload now happens as part of the new `Initialize` lifecycle phase. `zpm "<module> reload -only"` will no longer auto compile resources in `/preload` directory.
+- #726 When running `zpm "load ..."` on a nonexistent path, it now returns an error instead of silently failing with a $$$OK status.
 
 ### Fixed
 - #474: When loading a .tgz/.tar.gz package, automatically locate the top-most module.xml in case there is nested directory structure (e.g., GitHub releases)
 - #635: When calling the "package" command, the directory is now normalized to include trailing slash (or backslash).
 - #696: Fix a bug that caused error status to be ignored when publishing a module.
 - #700: Fix a bug due to incompatible conventions between SemVer and OCI tags
 - #669: Work with a wider variety of ORAS repos (removes _catalog call)
+- #726: Fixed a bug where loading a tarball doesn't install dependencies from `.modules` subfolder even when it's available
 
 ### Security
 -

src/cls/IPM/Main.cls

@@ -73,7 +73,7 @@ Currently, there may only be one of these per namespace.
 <modifier name="dev" dataAlias="DeveloperMode" dataValue="1" description="Sets the DeveloperMode flag for the module's lifecycle. Key consequences of this are that ^Sources will be configured for resources in the module, and installer methods will be called with the dev mode flag set." />
 <modifier name="quiet" aliases="q" dataAlias="Verbose" dataValue="0" description="Produces minimal output from the command." />
 <modifier name="verbose" aliases="v" dataAlias="Verbose" dataValue="1" description="Produces verbose output from the command." />
-<modifier name="export-deps" value="true" dataAlias="ExportDependencies" description="If specified, controls whether dependencies are exported. If omitted, defaults to the value of the #EXPORTDEPENDENCIES in lifecycle class. This modifier is only used in &quot;Package&quot; lifecycle." />
+<modifier name="export-deps" value="true" valueList="0,1" dataAlias="ExportDependencies" description="If specified, controls whether dependencies are exported. If omitted, defaults to the value of the #EXPORTDEPENDENCIES in lifecycle class. This modifier is only used in &quot;Package&quot; lifecycle." />
 
 </command>
 
@@ -139,6 +139,7 @@ This command is an alias for `module-action module-name package`
 <modifier name="quiet" aliases="q" dataAlias="Verbose" dataValue="0" description="Produces minimal output from the command." />
 <modifier name="verbose" aliases="v" dataAlias="Verbose" dataValue="1" description="Produces verbose output from the command." />
 <modifier name="path" aliases="p" dataAlias="Path" value="true" description="Use specified path to export package." />
+<modifier name="export-deps" value="true" valueList="0,1" dataAlias="ExportDependencies" description="If specified, controls whether dependencies are exported. If omitted, defaults to the value of the #EXPORTDEPENDENCIES in lifecycle class." />
 </command>
 
 <command name="verify" dataPrefix="D">
@@ -2074,7 +2075,6 @@ ClassMethod Load(ByRef pCommandInfo) [ Internal ]
 	}
 	If ##class(%File).DirectoryExists(tDirectoryName) {
 		Set tParams("DeveloperMode") = $Get(tParams("DeveloperMode"), 1)
-		$$$ThrowOnError(##class(%IPM.Utils.Module).LoadNewModule(tDirectoryName,.tParams))
 	} ElseIf ##class(%File).Exists(tDirectoryName) && (($$$lcase($Piece(tDirectoryName,".", *))="tgz") || ($$$lcase($Piece(tDirectoryName,".", *-1, *))="tar.gz")) {
 		Set tTargetDirectory = $$$FileTempDirSys
 		If $Get(pCommandInfo("data", "Verbose")) {
@@ -2088,23 +2088,48 @@ ClassMethod Load(ByRef pCommandInfo) [ Internal ]
 		Set tCount = ##class(%IPM.Utils.File).FindFiles(tTargetDirectory, "module.xml", .tList)
 		If (tCount = 0) {
 			$$$ThrowStatus($$$ERROR($$$GeneralError,"No module.xml file found in archive."))
-		} Else {
-			// The top-level module.xml is the last one in tList, assuming ##class(%File).FileSetFunc returns directory first
-			Set tModuleFilePath = $ListGet(tList(tCount), *)
-			Set tTargetDirectory = ##class(%File).GetDirectory(tModuleFilePath, 1)
-			If $Get(pCommandInfo("data", "Verbose")) {
-				Write !,"Found "_tCount_" results. They are: "
-				Set idx = ""
-				For {
-					Set idx = $Order(tList(idx))
-					Quit:(idx="")
-					Write !, $Char(9), $List(tList(idx), *)
-				}
-				Write !, "Using module.xml file at ", tModuleFilePath
+		}
+
+		// The top-level module.xml is the last one in tList, assuming ##class(%File).FileSetFunc returns directory first
+		Set tModuleFilePath = $ListGet(tList(tCount), *)
+		Set tTargetDirectory = ##class(%File).GetDirectory(tModuleFilePath, 1)
+		If $Get(pCommandInfo("data", "Verbose")) {
+			Write !,"Found "_tCount_" results. They are: "
+			Set idx = ""
+			For {
+				Set idx = $Order(tList(idx))
+				Quit:(idx="")
+				Write !, $Char(9), $List(tList(idx), *)
+			}
+			Write !, "Using module.xml file at ", tModuleFilePath
+		}
+	}
+
+	// When loading a module from a local folder, there might be a <mod root>/.modules/ folder containining dependencies.
+	// It's easier to configure a temporary repository than to handle this case in the dependency resolution code.
+	Set tTargetDirectory = $Get(tTargetDirectory, tDirectoryName)
+	Set dotModules = ##class(%File).NormalizeDirectory(".modules", tTargetDirectory)
+	If (dotModules '= "") && ##class(%File).DirectoryExists(dotModules) {
+		Set count = 0
+		For {
+			Set repoName = "ipm-temp-dot-modules-" _ $Increment(count)
+			If '##class(%IPM.Repo.Definition).ServerDefinitionKeyExists(repoName) {
+				Quit
 			}
 		}
-		$$$ThrowOnError(##class(%IPM.Utils.Module).LoadNewModule(tTargetDirectory, .tParams))
+		Set tempRepo = ##class(%IPM.Repo.Filesystem.Definition).%New()
+		Set tempRepo.Name = repoName
+		Set tempRepo.Root = dotModules
+		Set tempRepo.Snapshots = 1
+		Set tempRepo.Prereleases = 1
+		Set tempRepo.OverriddenSortOrder = -1000 // Make sure this is the first repo to be found by SQL query in %IPM.Repo.Manager:SearchRepositoriesForModule
+		$$$ThrowOnError(tempRepo.BuildCache(1,1,1))
+	}
+	Set tSC = ##class(%IPM.Utils.Module).LoadNewModule(tTargetDirectory, .tParams)
+	If $Data(tempRepo) # 2 {
+		Set tSC = $$$ADDSC(tSC, tempRepo.%DeleteId(tempRepo.%Id()))
 	}
+	$$$ThrowOnError(tSC)
 }
 
 ClassMethod CheckModuleNamespace() As %Status

src/cls/IPM/Repo/Definition.cls

@@ -31,6 +31,11 @@ Property Snapshots As %Boolean [ InitialExpression = 0 ];
 
 Property Prereleases As %Boolean [ InitialExpression = 0 ];
 
+/// If set to a non-empty value, this will be used as the sort order for this repository, bypassing the GetSortOrder() method.
+/// This should be only configured by IPM during certain processes (e.g., installing depdenencies from .modules directory).
+/// There is no current plan to support configuring this from the package manager shell.
+Property OverriddenSortOrder As %Integer [ Internal ];
+
 ClassMethod %GetCommandStructure(Output pCommandStructure)
 {
   Kill pCommandStructure
@@ -80,6 +85,9 @@ Method GetPublishService() As %IPM.Repo.IPublishService [ Abstract ]
 ClassMethod SortOrder(pID As %String) As %Integer [ SqlProc ]
 {
 	Set tServer = ..%OpenId(pID)
+	If tServer.OverriddenSortOrder '= "" {
+		Quit tServer.OverriddenSortOrder
+	}
 	Quit tServer.GetSortOrder()
 }
 
@@ -283,6 +291,9 @@ Storage Default
 <Value name="12">
 <Value>ReadOnly</Value>
 </Value>
+<Value name="13">
+<Value>OverriddenSortOrder</Value>
+</Value>
 </Data>
 <DataLocation>^IPM.Repo.DefinitionD</DataLocation>
 <DefaultData>RepoDefinitionDefaultData</DefaultData>

tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls

@@ -28,7 +28,7 @@ Method TestPackageAndExtract()
     Do $$$AssertTrue(##class(%File).Exists(outDir_"src/cls/Test/Test.cls"))
     Do $$$AssertTrue(##class(%File).Exists(outDir_"module.xml"))
 
-    Set tSC = ##class(%IPM.Main).Shell("load "_outFile)
+    Set tSC = ##class(%IPM.Main).Shell("load "_tempDir_".tgz")
     Do $$$AssertStatusOK(tSC,"Loaded SimpleModule module successfully from .tgz file.")
 
     Set tSC = ##class(%IPM.Main).Shell("load "_outDir)

tests/integration_tests/Test/PM/Integration/IPMVersionReq.cls

@@ -7,7 +7,7 @@ Method TestInteroperabilityEnabled()
 {
   Set tCases = $ListBuild( 
     $ListBuild("version-req-missing", 1),
-    $ListBuild("version-req-satisified", 1),
+    $ListBuild("version-req-satisfied", 1),
     $ListBuild("version-req-too-high", 0),
     $ListBuild("version-req-too-low", 0)
   )

tests/integration_tests/Test/PM/Integration/Interoperability.cls

@@ -6,7 +6,7 @@ Parameter CommonPathPrefix As STRING = "interoperability-test";
 Method TestInteroperabilityDisabled()
 {
   Set subfolder = "interoperability-disabled"
-  Set tModuleDir = ..GetModuleDir(subfolder)
+  Set tModuleDir = ..GetModuleDir(..#CommonPathPrefix, subfolder)
   Set tSC = ##class(%IPM.Main).Shell("load " _ tModuleDir)
   Do $$$AssertStatusOK(tSC,"Loaded module successfully(" _ subfolder _ ")")
 }

tests/integration_tests/Test/PM/Integration/LoadModuleWithDeps.cls

@@ -0,0 +1,30 @@
+Class Test.PM.Integration.LoadModuleWithDeps Extends Test.PM.Integration.Base
+{
+
+Parameter ModuleFolder = "load-export";
+
+Method TestLoadModuleWithDependencies() As %Status
+{
+    /// Load from folder
+    Set moduleDir = ..GetModuleDir(..#ModuleFolder)
+    Set sc = ##class(%IPM.Main).Shell("load -v " _ moduleDir)
+    Do $$$AssertStatusOK(sc, "Successfully load from folder")
+
+    /// Load from tarball
+    Set tempDir = $$$FileTempDir
+    Set exportPath = ##class(%File).NormalizeFilename("package", tempDir)
+    Set tarballPath = ##class(%File).NormalizeFilename("package.tgz", tempDir)
+
+    Set sc = ##class(%IPM.Main).Shell("package load-export -v -export-deps 1 -path " _ exportPath)
+    Do $$$AssertStatusOK(sc, "Successfully export package as tarball")
+    Set sc = ##class(%IPM.Main).Shell("uninstall load-export-deps")
+    Do $$$AssertStatusOK(sc, "Uninstalled load-export-deps")
+    Set sc = ##class(%IPM.Main).Shell("uninstall load-export")
+    Do $$$AssertStatusOK(sc, "Uninstalled load-export")
+    Set sc = ##class(%IPM.Main).Shell("load -v " _ tarballPath)
+    Do $$$AssertStatusOK(sc, "Successfully load from tarball")
+
+    Quit $$$OK
+}
+
+}

tests/integration_tests/Test/PM/Integration/_data/interoperability-test/interoperability-disabled/module.xml

@@ -6,7 +6,7 @@
       <Version>0.0.1+snapshot</Version>
       <Packaging>module</Packaging>
       <SourcesRoot>src</SourcesRoot>
-      <SystemRequirements Version=">=2020.1" Interoperability="disabled" Health="false" />
+      <SystemRequirements Version=">=2020.1" Interoperability="disabled"/>
     </Module>
   </Document>
 </Export>

tests/integration_tests/Test/PM/Integration/_data/ipm-version-req-test/version-req-satisfied/module.xml

@@ -5,7 +5,7 @@
       <Name>version-req-satisfied</Name>
       <Version>0.0.1+snapshot</Version>
       <Packaging>module</Packaging>
-      <SystemRequirements IPMVersion=">=0.9.0-0"/>
+      <SystemRequirements IPMVersion="*"/>
     </Module>
   </Document>
 </Export>

tests/integration_tests/Test/PM/Integration/_data/load-export/.modules/dep/module.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Export generator="Cache" version="25">
+  <Document name="load-export-deps.ZPM">
+    <Module>
+      <Name>load-export-deps</Name>
+      <Version>0.0.1</Version>
+      <Packaging>module</Packaging>
+    </Module>
+  </Document>
+</Export>

tests/integration_tests/Test/PM/Integration/_data/load-export/module.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Export generator="Cache" version="25">
+  <Document name="load-export.ZPM">
+    <Module>
+      <Name>load-export</Name>
+      <Version>0.0.1</Version>
+      <Packaging>module</Packaging>
+      <Dependencies>
+        <ModuleReference>
+          <Name>load-export-deps</Name>
+          <Version>0.0.1</Version>
+        </ModuleReference>
+      </Dependencies>
+    </Module>
+  </Document>
+</Export>