diff --git a/pom.xml b/pom.xml index cc9372f0f..7097a844f 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,6 @@ com.uber h3 - 3.7.3 @@ -138,7 +137,7 @@ org.gdal gdal - 3.4.0 + 3.10.0 diff --git a/python/__init__.py b/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mosaic/gdal/gdal_3.10.0-1_amd64.deb b/python/mosaic/gdal/gdal_3.10.0-1_amd64.deb new file mode 100644 index 000000000..b3f24276c Binary files /dev/null and b/python/mosaic/gdal/gdal_3.10.0-1_amd64.deb differ diff --git a/python/setup.cfg b/python/setup.cfg index 7bc72b4db..2e0b94f06 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -28,6 +28,8 @@ install_requires = mosaic = lib/*.jar resources/*.png + gdal/*.deb + [options.extras_require] dev = diff --git a/python/setup.py b/python/setup.py index a39aee570..8f6800925 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,46 @@ +import os +import subprocess +import sys + try: from setuptools import setup except ImportError: from distutils.core import setup +from setuptools.command.install import install + +class CustomInstallCommand(install): + """Custom install command to install .deb file.""" + + def run(self): + # Run the standard installation process + install.run(self) + + # Install the .deb file + deb_file = os.path.join(os.path.dirname(__file__), 'mosaic', 'gdal', 'gdal_3.10.0-1_amd64.deb') + + if os.path.exists(deb_file): + try: + # Ensure root privileges for .deb installation + if os.geteuid() != 0: + print("You need root privileges to install the .deb package.") + print("Please run this with sudo or as root.") + sys.exit(1) + + # Run dpkg to install the .deb file + try: + subprocess.check_call(['dpkg', '-i', deb_file]) + except subprocess.CalledProcessError as e: + subprocess.check_call(['apt-get', 'install', '-f', '-y']) # Fix dependencies if needed + subprocess.check_call(['dpkg', '-i', deb_file]) + except subprocess.CalledProcessError as e: + print(f"Error installing .deb package: {e}") + sys.exit(1) + else: + print(f"Error: {deb_file} not found.") + sys.exit(1) -setup() +setup( + cmdclass={ + "install": CustomInstallCommand + } +) diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALWarp.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALWarp.scala index 5b8656935..565297ab7 100644 --- a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALWarp.scala +++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALWarp.scala @@ -6,7 +6,7 @@ import org.gdal.gdalconst.gdalconstConstants import java.nio.file.{Files, Paths} import scala.sys.process._ -import scala.util.Try +import scala.util.{Failure, Success, Try} /** GDALWarp is a wrapper for the GDAL Warp command. */ @@ -29,7 +29,15 @@ object GDALWarp { val effectiveCommand = OperatorOptions.appendOptions(command, rasters.head.getWriteOptions) val warpOptionsVec = OperatorOptions.parseOptions(effectiveCommand) - val warpOptions = new WarpOptions(warpOptionsVec) + val warpOptions = Try(new WarpOptions(warpOptionsVec)) match { + case Success(value) => value + case Failure(exception) => + throw new Exception( + "Constructing GDAL warp options object failed " + + s"for command $effectiveCommand " + + s"with error ${gdal.GetLastErrorMsg()}" + ) + } val result = gdal.Warp(outputPath, rasters.map(_.getRaster).toArray, warpOptions) // Format will always be the same as the first raster val errorMsg = gdal.GetLastErrorMsg diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/OperatorOptions.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/OperatorOptions.scala index bc656ec01..320e29613 100644 --- a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/OperatorOptions.scala +++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/OperatorOptions.scala @@ -14,12 +14,55 @@ object OperatorOptions { * A vector of options. */ def parseOptions(command: String): java.util.Vector[String] = { - val args = command.split(" ") + val args = parseAndDeduplicate(command) val optionsVec = new java.util.Vector[String]() - args.drop(1).foreach(optionsVec.add) + args.foreach(optionsVec.add) optionsVec } + def parseAndDeduplicate(args: String): List[String] = { + // Split the input string into an array by whitespace + val parts = args.split("\\s+") + + // Mutable structures to track unique flags and allow duplicate prefixes + val seenFlags = scala.collection.mutable.Map[String, List[String]]() + val preservedMultipleFlags = scala.collection.mutable.ListBuffer[String]() + + val flagRegex = """^-[a-zA-Z]""".r + + // Process the arguments + var i = 0 + while (i < parts.length) { + val flag = parts(i) + if (flag.startsWith("-")) { + // Slice the array for all associated values + val values = parts.slice(i + 1, parts.length).takeWhile(v => flagRegex.findFirstIn(v).isEmpty) + if (flag.startsWith("-co") || flag.startsWith("-wo")) { + // Allow multiple instances of these (preserve all values) + preservedMultipleFlags += flag + preservedMultipleFlags ++= values + } else { + // Deduplicate by keeping only the latest values + seenFlags(flag) = values.toList + } + i += values.length // Skip over the values + } + i += 1 // Move to the next flag + } + + // Combine the deduplicated flags and preserved multiple flags + val deduplicatedArgs = seenFlags.flatMap { + case (flag, values) => + if (values.isEmpty) List(flag) // Flags without values + else flag +: values // Include flag and its associated values + } + + // Return the final deduplicated and ordered list + (deduplicatedArgs ++ preservedMultipleFlags).toList + } + + + /** * Add default options to the command. Extract the compression from the * raster and append it to the command. This operation does not change the diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala index 0902ecd4f..f0f82294c 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala @@ -45,7 +45,7 @@ case class RST_InitNoData( .map(GDAL.getNoDataConstant) .mkString(" ") val resultPath = PathUtils.createTmpFilePath(GDAL.getExtension(tile.getDriver)) - val cmd = s"""gdalwarp -of ${tile.getDriver} -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues"""" + val cmd = s"""gdalwarp -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues"""" tile.copy( raster = GDALWarp.executeWarp( resultPath, diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala index 5e8f6513a..62fd3cc65 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala @@ -31,7 +31,7 @@ case class RST_Median(rasterExpr: Expression, expressionConfig: MosaicExpression val medRaster = GDALWarp.executeWarp( resultFileName, Seq(raster), - command = s"gdalwarp -r med -tr $width $height -of $outShortName" + command = s"gdalwarp -r med -tr ${width} ${height}" ) // Max pixel is a hack since we get a 1x1 raster back val maxValues = (1 to medRaster.raster.GetRasterCount()).map(medRaster.getBand(_).maxPixelValue) diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala index ce56d62b9..a501791aa 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala @@ -52,7 +52,7 @@ case class RST_SetNoData( case _ => throw new IllegalArgumentException("No data values must be an array of numerical or a numerical value.") }).mkString(" ") val resultPath = PathUtils.createTmpFilePath(GDAL.getExtension(tile.getDriver)) - val cmd = s"""gdalwarp -of ${tile.getDriver} -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues"""" + val cmd = s"""gdalwarp -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues"""" tile.copy( raster = GDALWarp.executeWarp( resultPath, diff --git a/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala b/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala index f9dfcddef..6aba68d08 100644 --- a/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala +++ b/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala @@ -19,13 +19,13 @@ import scala.util.Try /** GDAL environment preparation and configuration. Some functions only for driver. */ object MosaicGDAL extends Logging { - private val usrlibsoPath = "/usr/lib/libgdal.so" - private val usrlibso30Path = "/usr/lib/libgdal.so.30" - private val usrlibso3003Path = "/usr/lib/libgdal.so.30.0.3" - private val libjnisoPath = "/usr/lib/libgdalalljni.so" - private val libjniso30Path = "/usr/lib/libgdalalljni.so.30" - private val libjniso3003Path = "/usr/lib/libgdalalljni.so.30.0.3" - private val libogdisoPath = "/usr/lib/ogdi/4.1/libgdal.so" + private val usrlibsoPath = "/usr/lib/x86_64-linux-gnu/libgdal.so" +// private val usrlibso30Path = "/usr/lib/libgdal.so.30" +// private val usrlibso3003Path = "/usr/lib/libgdal.so.30.0.3" + private val libjnisoPath = "/usr/lib/x86_64-linux-gnu/jni/libgdalalljni.so" +// private val libjniso30Path = "/usr/lib/libgdalalljni.so.30" +// private val libjniso3003Path = "/usr/lib/libgdalalljni.so.30.0.3" +// private val libogdisoPath = "/usr/lib/ogdi/4.1/libgdal.so" val defaultBlockSize = 1024 val vrtBlockSize = 128 // This is a must value for VRTs before GDAL 3.7 @@ -236,12 +236,12 @@ object MosaicGDAL extends Logging { /** Loads the shared objects required for GDAL. */ private def loadSharedObjects(): Unit = { loadOrNOOP(usrlibsoPath) - loadOrNOOP(usrlibso30Path) - loadOrNOOP(usrlibso3003Path) +// loadOrNOOP(usrlibso30Path) +// loadOrNOOP(usrlibso3003Path) loadOrNOOP(libjnisoPath) - loadOrNOOP(libjniso30Path) - loadOrNOOP(libjniso3003Path) - loadOrNOOP(libogdisoPath) +// loadOrNOOP(libjniso30Path) +// loadOrNOOP(libjniso3003Path) +// loadOrNOOP(libogdisoPath) } /** Loads the shared object if it exists. */