Add ZStd compression support for GTiff (#3580)

* Add ZStd compression support for GTiff

* cleanup the code / build

* ZStdDecompressor constructor cleanup

---------

Co-authored-by: dsamaey <[email protected]>
Co-authored-by: Grigory Pomadchin <[email protected]>

Author: dsamaey

CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Add ZStd compression support for GTiff
+
 ## [3.8.0] - 2025-04-23
 
 ### Added

project/Dependencies.scala

@@ -92,6 +92,7 @@ object Dependencies {
   val sl4jnop             = "org.slf4j"                   % "slf4j-nop"                % "1.7.25"
   val logbackClassic      = "ch.qos.logback"              % "logback-classic"          % "1.2.3"
   val guava               = "com.google.guava"            % "guava"                    % "16.0.1"
+  val zstdJni             = "com.github.luben"            % "zstd-jni"                 % "1.5.7-3"
 
   val cassandraDriverCore         = "com.datastax.oss" % "java-driver-core"          % Version.cassandra
   val cassandraDriverQueryBuilder = "com.datastax.oss" % "java-driver-query-builder" % Version.cassandra

project/Settings.scala

@@ -40,10 +40,9 @@ object Settings {
     val ivy2Local         = Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
     val mavenLocal        = Resolver.mavenLocal
     val maven             = DefaultMavenRepository
-    val sonatypeReleases  = Resolver.sonatypeOssRepos("releases") 
-    val sonatypeSnapshots = Resolver.sonatypeOssRepos("snapshots")
+    val sonatypeSnapshots = Resolver.sonatypeCentralSnapshots
     val local             = Seq(ivy2Local, mavenLocal)
-    val external          = Seq(osgeoReleases, maven, eclipseReleases, geosolutions, jitpack, apacheSnapshots, osgeoSnapshots) ++ sonatypeReleases ++ sonatypeSnapshots
+    val external          = Seq(osgeoReleases, maven, eclipseReleases, geosolutions, jitpack, apacheSnapshots, osgeoSnapshots, sonatypeSnapshots)
     val all               = external ++ local
   }
 
@@ -419,6 +418,8 @@ object Settings {
     name := "geotrellis-raster",
     libraryDependencies ++= Seq(
       squants,
+      zstdJni,
+      apacheIO excludeAll(ExclusionRule("com.fasterxml.jackson.core"), ExclusionRule("com.google.j2objc")),
       monocle("core").value,
       monocle("macro").value,
       scalaXml,

raster/src/main/scala/geotrellis/raster/io/geotiff/compression/Decompressor.scala

@@ -117,6 +117,8 @@ object Decompressor {
         checkEndian(PackBitsDecompressor(segmentSizes))
       case JpegCoded =>
         checkEndian(JpegDecompressor(tiffTags))
+      case ZStdCoded =>
+        checkPredictor(ZStdCompression.createDecompressor)
 
       // Unsupported compression types
       case HuffmanCoded =>

raster/src/main/scala/geotrellis/raster/io/geotiff/compression/ZStdCompression.scala

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2016 Azavea
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package geotrellis.raster.io.geotiff.compression
+
+import geotrellis.raster.io.geotiff.tags.codes.CompressionType._
+
+import com.github.luben.zstd.{ZstdInputStream, ZstdOutputStream}
+import org.apache.commons.io.IOUtils
+
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+
+case class ZStdCompression(level: Int = 3) extends Compression {
+  def createCompressor(segmentCount: Int): Compressor =
+    new ZStdCompressor(segmentCount, level)
+
+  def createDecompressor: Decompressor =
+    new ZStdDecompressor()
+
+}
+
+object ZStdCompression extends ZStdCompression(3)
+
+class ZStdCompressor(segmentCount: Int, level: Int) extends Compressor {
+  private val segmentSizes = Array.ofDim[Int](segmentCount)
+
+  def compress(segment: Array[Byte], segmentIndex: Int): Array[Byte] = {
+    segmentSizes(segmentIndex) = segment.size
+
+    val outputStream = new ByteArrayOutputStream()
+    val compressorOutputStream = new ZstdOutputStream(outputStream, level)
+    IOUtils.copyLarge(new ByteArrayInputStream(segment), compressorOutputStream)
+    compressorOutputStream.close()
+    outputStream.toByteArray
+  }
+
+  def createDecompressor(): Decompressor =
+    new ZStdDecompressor()
+}
+
+class ZStdDecompressor extends Decompressor {
+  def code = ZStdCoded
+
+  def decompress(segment: Array[Byte], segmentIndex: Int): Array[Byte] = {
+    val outputStream = new ByteArrayOutputStream()
+    val stream = new ByteArrayInputStream(segment)
+    val compressorInputStream = new ZstdInputStream(stream)
+    IOUtils.copyLarge(compressorInputStream, outputStream)
+    compressorInputStream.close()
+    outputStream.toByteArray
+  }
+}
+

raster/src/main/scala/geotrellis/raster/io/geotiff/tags/codes/CompressionType.scala

@@ -28,5 +28,6 @@ object CompressionType {
   val ZLibCoded = 8
   val PackBitsCoded = 32773
   val PkZipCoded = 32946
+  val ZStdCoded = 50000
 
 }

raster/src/test/scala/geotrellis/raster/io/geotiff/compression/ZStdCompressionSpec.scala

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2016 Azavea
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package geotrellis.raster.io.geotiff.compression
+
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.funspec.AnyFunSpec
+
+class ZStdCompressionSpec extends AnyFunSpec with Matchers {
+  describe("ZStdCompression") {
+    (-1 to 9).foreach { level =>
+      it(s"should decompress and compress to the same things at the ${if(level == -1) "default compression level" else s"zoom level $level"}") {
+        val segment =
+          """
+            | the human machine only moves when the tide of every one of us pushes to the outside
+            | and we realize that from the time you wake until the time you shake it up
+            | you're nothing more than a ghost echo
+            | it burns brighter when the pieces move the best way,
+            | whatever best may mean,
+            | it's so subjective. but when enough agree we can finally move it.
+            | when the pieces understand what they're doing they can chose a direction
+            | and run a little hotter.
+            | against inertia,
+            | it goes.
+            | whole lives change
+            | only the moment remains here""".getBytes("UTF-8")
+        val compressor = new ZStdCompression(level).createCompressor(1)
+        val compressed = compressor.compress(segment, 0)
+        val decompressor = compressor.createDecompressor()
+        val decompressed = decompressor.decompress(compressed, 0)
+        decompressed should be(segment)
+      }
+    }
+  }
+}