diff --git a/pom.xml b/pom.xml
index cc9372f0f..7097a844f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -111,7 +111,6 @@
com.uber
h3
-
3.7.3
@@ -138,7 +137,7 @@
org.gdal
gdal
- 3.4.0
+ 3.10.0
diff --git a/python/__init__.py b/python/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/python/mosaic/gdal/gdal_3.10.0-1_amd64.deb b/python/mosaic/gdal/gdal_3.10.0-1_amd64.deb
new file mode 100644
index 000000000..b3f24276c
Binary files /dev/null and b/python/mosaic/gdal/gdal_3.10.0-1_amd64.deb differ
diff --git a/python/setup.cfg b/python/setup.cfg
index 7bc72b4db..2e0b94f06 100644
--- a/python/setup.cfg
+++ b/python/setup.cfg
@@ -28,6 +28,8 @@ install_requires =
mosaic =
lib/*.jar
resources/*.png
+ gdal/*.deb
+
[options.extras_require]
dev =
diff --git a/python/setup.py b/python/setup.py
index a39aee570..8f6800925 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,6 +1,46 @@
+import os
+import subprocess
+import sys
+
try:
from setuptools import setup
except ImportError:
from distutils.core import setup
+from setuptools.command.install import install
+
+class CustomInstallCommand(install):
+ """Custom install command to install .deb file."""
+
+ def run(self):
+ # Run the standard installation process
+ install.run(self)
+
+ # Install the .deb file
+ deb_file = os.path.join(os.path.dirname(__file__), 'mosaic', 'gdal', 'gdal_3.10.0-1_amd64.deb')
+
+ if os.path.exists(deb_file):
+ try:
+ # Ensure root privileges for .deb installation
+ if os.geteuid() != 0:
+ print("You need root privileges to install the .deb package.")
+ print("Please run this with sudo or as root.")
+ sys.exit(1)
+
+ # Run dpkg to install the .deb file
+ try:
+ subprocess.check_call(['dpkg', '-i', deb_file])
+ except subprocess.CalledProcessError as e:
+ subprocess.check_call(['apt-get', 'install', '-f', '-y']) # Fix dependencies if needed
+ subprocess.check_call(['dpkg', '-i', deb_file])
+ except subprocess.CalledProcessError as e:
+ print(f"Error installing .deb package: {e}")
+ sys.exit(1)
+ else:
+ print(f"Error: {deb_file} not found.")
+ sys.exit(1)
-setup()
+setup(
+ cmdclass={
+ "install": CustomInstallCommand
+ }
+)
diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALWarp.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALWarp.scala
index 5b8656935..565297ab7 100644
--- a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALWarp.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALWarp.scala
@@ -6,7 +6,7 @@ import org.gdal.gdalconst.gdalconstConstants
import java.nio.file.{Files, Paths}
import scala.sys.process._
-import scala.util.Try
+import scala.util.{Failure, Success, Try}
/** GDALWarp is a wrapper for the GDAL Warp command. */
@@ -29,7 +29,15 @@ object GDALWarp {
val effectiveCommand = OperatorOptions.appendOptions(command, rasters.head.getWriteOptions)
val warpOptionsVec = OperatorOptions.parseOptions(effectiveCommand)
- val warpOptions = new WarpOptions(warpOptionsVec)
+ val warpOptions = Try(new WarpOptions(warpOptionsVec)) match {
+ case Success(value) => value
+ case Failure(exception) =>
+ throw new Exception(
+ "Constructing GDAL warp options object failed " +
+ s"for command $effectiveCommand " +
+ s"with error ${gdal.GetLastErrorMsg()}"
+ )
+ }
val result = gdal.Warp(outputPath, rasters.map(_.getRaster).toArray, warpOptions)
// Format will always be the same as the first raster
val errorMsg = gdal.GetLastErrorMsg
diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/OperatorOptions.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/OperatorOptions.scala
index bc656ec01..320e29613 100644
--- a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/OperatorOptions.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/OperatorOptions.scala
@@ -14,12 +14,55 @@ object OperatorOptions {
* A vector of options.
*/
def parseOptions(command: String): java.util.Vector[String] = {
- val args = command.split(" ")
+ val args = parseAndDeduplicate(command)
val optionsVec = new java.util.Vector[String]()
- args.drop(1).foreach(optionsVec.add)
+ args.foreach(optionsVec.add)
optionsVec
}
+ def parseAndDeduplicate(args: String): List[String] = {
+ // Split the input string into an array by whitespace
+ val parts = args.split("\\s+")
+
+ // Mutable structures to track unique flags and allow duplicate prefixes
+ val seenFlags = scala.collection.mutable.Map[String, List[String]]()
+ val preservedMultipleFlags = scala.collection.mutable.ListBuffer[String]()
+
+ val flagRegex = """^-[a-zA-Z]""".r
+
+ // Process the arguments
+ var i = 0
+ while (i < parts.length) {
+ val flag = parts(i)
+ if (flag.startsWith("-")) {
+ // Slice the array for all associated values
+ val values = parts.slice(i + 1, parts.length).takeWhile(v => flagRegex.findFirstIn(v).isEmpty)
+ if (flag.startsWith("-co") || flag.startsWith("-wo")) {
+ // Allow multiple instances of these (preserve all values)
+ preservedMultipleFlags += flag
+ preservedMultipleFlags ++= values
+ } else {
+ // Deduplicate by keeping only the latest values
+ seenFlags(flag) = values.toList
+ }
+ i += values.length // Skip over the values
+ }
+ i += 1 // Move to the next flag
+ }
+
+ // Combine the deduplicated flags and preserved multiple flags
+ val deduplicatedArgs = seenFlags.flatMap {
+ case (flag, values) =>
+ if (values.isEmpty) List(flag) // Flags without values
+ else flag +: values // Include flag and its associated values
+ }
+
+ // Return the final deduplicated and ordered list
+ (deduplicatedArgs ++ preservedMultipleFlags).toList
+ }
+
+
+
/**
* Add default options to the command. Extract the compression from the
* raster and append it to the command. This operation does not change the
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala
index 0902ecd4f..f0f82294c 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala
@@ -45,7 +45,7 @@ case class RST_InitNoData(
.map(GDAL.getNoDataConstant)
.mkString(" ")
val resultPath = PathUtils.createTmpFilePath(GDAL.getExtension(tile.getDriver))
- val cmd = s"""gdalwarp -of ${tile.getDriver} -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues""""
+ val cmd = s"""gdalwarp -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues""""
tile.copy(
raster = GDALWarp.executeWarp(
resultPath,
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala
index 5e8f6513a..62fd3cc65 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala
@@ -31,7 +31,7 @@ case class RST_Median(rasterExpr: Expression, expressionConfig: MosaicExpression
val medRaster = GDALWarp.executeWarp(
resultFileName,
Seq(raster),
- command = s"gdalwarp -r med -tr $width $height -of $outShortName"
+ command = s"gdalwarp -r med -tr ${width} ${height}"
)
// Max pixel is a hack since we get a 1x1 raster back
val maxValues = (1 to medRaster.raster.GetRasterCount()).map(medRaster.getBand(_).maxPixelValue)
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala
index ce56d62b9..a501791aa 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala
@@ -52,7 +52,7 @@ case class RST_SetNoData(
case _ => throw new IllegalArgumentException("No data values must be an array of numerical or a numerical value.")
}).mkString(" ")
val resultPath = PathUtils.createTmpFilePath(GDAL.getExtension(tile.getDriver))
- val cmd = s"""gdalwarp -of ${tile.getDriver} -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues""""
+ val cmd = s"""gdalwarp -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues""""
tile.copy(
raster = GDALWarp.executeWarp(
resultPath,
diff --git a/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala b/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala
index f9dfcddef..6aba68d08 100644
--- a/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala
@@ -19,13 +19,13 @@ import scala.util.Try
/** GDAL environment preparation and configuration. Some functions only for driver. */
object MosaicGDAL extends Logging {
- private val usrlibsoPath = "/usr/lib/libgdal.so"
- private val usrlibso30Path = "/usr/lib/libgdal.so.30"
- private val usrlibso3003Path = "/usr/lib/libgdal.so.30.0.3"
- private val libjnisoPath = "/usr/lib/libgdalalljni.so"
- private val libjniso30Path = "/usr/lib/libgdalalljni.so.30"
- private val libjniso3003Path = "/usr/lib/libgdalalljni.so.30.0.3"
- private val libogdisoPath = "/usr/lib/ogdi/4.1/libgdal.so"
+ private val usrlibsoPath = "/usr/lib/x86_64-linux-gnu/libgdal.so"
+// private val usrlibso30Path = "/usr/lib/libgdal.so.30"
+// private val usrlibso3003Path = "/usr/lib/libgdal.so.30.0.3"
+ private val libjnisoPath = "/usr/lib/x86_64-linux-gnu/jni/libgdalalljni.so"
+// private val libjniso30Path = "/usr/lib/libgdalalljni.so.30"
+// private val libjniso3003Path = "/usr/lib/libgdalalljni.so.30.0.3"
+// private val libogdisoPath = "/usr/lib/ogdi/4.1/libgdal.so"
val defaultBlockSize = 1024
val vrtBlockSize = 128 // This is a must value for VRTs before GDAL 3.7
@@ -236,12 +236,12 @@ object MosaicGDAL extends Logging {
/** Loads the shared objects required for GDAL. */
private def loadSharedObjects(): Unit = {
loadOrNOOP(usrlibsoPath)
- loadOrNOOP(usrlibso30Path)
- loadOrNOOP(usrlibso3003Path)
+// loadOrNOOP(usrlibso30Path)
+// loadOrNOOP(usrlibso3003Path)
loadOrNOOP(libjnisoPath)
- loadOrNOOP(libjniso30Path)
- loadOrNOOP(libjniso3003Path)
- loadOrNOOP(libogdisoPath)
+// loadOrNOOP(libjniso30Path)
+// loadOrNOOP(libjniso3003Path)
+// loadOrNOOP(libogdisoPath)
}
/** Loads the shared object if it exists. */