feat: Standardized dictionary handling

Feature * Zstd.compress supports :level and :dict keyward args * Zstd.decompress supports :dict keyward args * Zstd::StreamingCompress.new supports :level and :dict keyward args * Zstd::StreamingDecompress.new supports :dict keyward args Breaking Change * Zstd.compress uses ZSTD_compressStream2 instead of ZSTD_compress * Zstd.decompress uses ZSTD_decompressDCtx instead of ZSTD_decompress Deprecation * Zstd.compress_using_dict adds deprecation warning * use Zstd.compress with :dict keyward args * Zstd.decompress_using_dict adds deprecation warning * use Zstd.decompress with :dict keyward args * Zstd.compress with level args add deprecation warning * use Zstd.compress with :level keyward args * Zstd::StreamingCompress.new with level args add deprecation warning * use Zstd::StreamingCompress.new with :level keyward args
SpringMT · Apr 11, 2024 · 0a753d2 · 0a753d2
1 parent cf9b448
commit 0a753d2
Show file tree

Hide file tree

Showing 16 changed files with 262 additions and 95 deletions.
diff --git a/benchmarks/results/city.json.gzip b/benchmarks/results/city.json.gzip
diff --git a/examples/sinatra/Gemfile b/examples/sinatra/Gemfile
@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "sinatra"
+gem "rackup"
+gem "zstd-ruby", path: "../../"
diff --git a/examples/sinatra/Gemfile.lock b/examples/sinatra/Gemfile.lock
@@ -0,0 +1,41 @@
+PATH
+  remote: ../..
+  specs:
+    zstd-ruby (1.5.6.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    base64 (0.2.0)
+    mustermann (3.0.0)
+      ruby2_keywords (~> 0.0.1)
+    rack (3.0.10)
+    rack-protection (4.0.0)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0, < 4)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
+    rackup (2.1.0)
+      rack (>= 3)
+      webrick (~> 1.8)
+    ruby2_keywords (0.0.5)
+    sinatra (4.0.0)
+      mustermann (~> 3.0)
+      rack (>= 3.0.0, < 4)
+      rack-protection (= 4.0.0)
+      rack-session (>= 2.0.0, < 3)
+      tilt (~> 2.0)
+    tilt (2.3.0)
+    webrick (1.8.1)
+
+PLATFORMS
+  arm64-darwin-21
+  ruby
+
+DEPENDENCIES
+  rackup
+  sinatra
+  zstd-ruby!
+
+BUNDLED WITH
+   2.5.7
diff --git a/examples/sinatra/app.rb b/examples/sinatra/app.rb
@@ -0,0 +1,7 @@
+require 'sinatra'
+require 'zstd-ruby'
+
+get '/' do
+  headers["Content-Encoding"] = "zstd"
+  Zstd.compress('Hello world!')
+end
diff --git a/ext/zstdruby/common.h b/ext/zstdruby/common.h
@@ -1,7 +1,7 @@
 #ifndef ZSTD_RUBY_H
 #define ZSTD_RUBY_H 1
 
-#include "ruby.h"
+#include <ruby.h>
 #include "./libzstd/zstd.h"
 
 static int convert_compression_level(VALUE compression_level_value)
@@ -12,4 +12,55 @@ static int convert_compression_level(VALUE compression_level_value)
   return NUM2INT(compression_level_value);
 }
 
+static size_t zstd_compress(ZSTD_CCtx* const ctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective endOp)
+{
+  return ZSTD_compressStream2(ctx, output, input, endOp);
+}
+
+static void set_compress_params(ZSTD_CCtx* const ctx, VALUE level_from_args, VALUE kwargs)
+{
+  ID kwargs_keys[2];
+  kwargs_keys[0] = rb_intern("level");
+  kwargs_keys[1] = rb_intern("dict");
+  VALUE kwargs_values[2];
+  rb_get_kwargs(kwargs, kwargs_keys, 0, 2, kwargs_values);
+
+  int compression_level = ZSTD_CLEVEL_DEFAULT;
+  if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
+    compression_level = convert_compression_level(kwargs_values[0]);
+  } else if (!NIL_P(level_from_args)) {
+    rb_warn("`level` in args is deprecated; use keyword args `level:` instead.");
+    compression_level = convert_compression_level(level_from_args);
+  }
+  ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
+
+  if (kwargs_values[1] != Qundef && kwargs_values[1] != Qnil) {
+    char* dict_buffer = RSTRING_PTR(kwargs_values[1]);
+    size_t dict_size = RSTRING_LEN(kwargs_values[1]);
+    size_t load_dict_ret = ZSTD_CCtx_loadDictionary(ctx, dict_buffer, dict_size);
+    if (ZSTD_isError(load_dict_ret)) {
+      ZSTD_freeCCtx(ctx);
+      rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
+    }
+  }
+}
+
+static void set_decompress_params(ZSTD_DCtx* const dctx, VALUE kwargs)
+{
+  ID kwargs_keys[1];
+  kwargs_keys[0] = rb_intern("dict");
+  VALUE kwargs_values[1];
+  rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
+
+  if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
+    char* dict_buffer = RSTRING_PTR(kwargs_values[0]);
+    size_t dict_size = RSTRING_LEN(kwargs_values[0]);
+    size_t load_dict_ret = ZSTD_DCtx_loadDictionary(dctx, dict_buffer, dict_size);
+    if (ZSTD_isError(load_dict_ret)) {
+      ZSTD_freeDCtx(dctx);
+      rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
+    }
+  }
+}
+
 #endif /* ZSTD_RUBY_H */
diff --git a/ext/zstdruby/main.c b/ext/zstdruby/main.c
@@ -1,4 +1,5 @@
-#include <common.h>
+#include "common.h"
+
 VALUE rb_mZstd;
 void zstd_ruby_init(void);
 void zstd_ruby_skippable_frame_init(void);

diff --git a/ext/zstdruby/skippable_frame.c b/ext/zstdruby/skippable_frame.c
@@ -1,4 +1,4 @@
-#include <common.h>
+#include "common.h"
 
 extern VALUE rb_mZstd;
 

diff --git a/ext/zstdruby/streaming_compress.c b/ext/zstdruby/streaming_compress.c
@@ -1,5 +1,4 @@
-#include <common.h>
-#include <streaming_compress.h>
+#include "common.h"
 
 struct streaming_compress_t {
   ZSTD_CCtx* ctx;
@@ -71,14 +70,9 @@ rb_streaming_compress_allocate(VALUE klass)
 static VALUE
 rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
 {
+  VALUE kwargs;
   VALUE compression_level_value;
-  rb_scan_args(argc, argv, "01", &compression_level_value);
-  int compression_level = convert_compression_level(compression_level_value);
-
-  ID kwargs_keys[1];
-  kwargs_keys[0] = rb_intern("dict");
-  VALUE kwargs_values[1];
-  rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
+  rb_scan_args(argc, argv, "01:", &compression_level_value, &kwargs);
 
   struct streaming_compress_t* sc;
   TypedData_Get_Struct(obj, struct streaming_compress_t, &streaming_compress_type, sc);
@@ -88,15 +82,8 @@ rb_streaming_compress_initialize(int argc, VALUE *argv, VALUE obj)
   if (ctx == NULL) {
     rb_raise(rb_eRuntimeError, "%s", "ZSTD_createCCtx error");
   }
-  if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
-    char* dict_buffer = RSTRING_PTR(kwargs_values[0]);
-    size_t dict_size = RSTRING_LEN(kwargs_values[0]);
-    size_t load_dict_ret = ZSTD_CCtx_loadDictionary(ctx, dict_buffer, dict_size);
-    if (ZSTD_isError(load_dict_ret)) {
-      rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
-    }
-  }
-  ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
+  set_compress_params(ctx, compression_level_value, kwargs);
+
   sc->ctx = ctx;
   sc->buf = rb_str_new(NULL, buffOutSize);
   sc->buf_size = buffOutSize;
@@ -119,7 +106,7 @@ no_compress(struct streaming_compress_t* sc, ZSTD_EndDirective endOp)
   do {
     ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
 
-    size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, endOp);
+    size_t const ret = zstd_compress(sc->ctx, &output, &input, endOp);
     if (ZSTD_isError(ret)) {
       rb_raise(rb_eRuntimeError, "flush error error code: %s", ZSTD_getErrorName(ret));
     }
@@ -143,7 +130,7 @@ rb_streaming_compress_compress(VALUE obj, VALUE src)
   VALUE result = rb_str_new(0, 0);
   while (input.pos < input.size) {
     ZSTD_outBuffer output = { (void*)output_data, sc->buf_size, 0 };
-    size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
+    size_t const ret = zstd_compress(sc->ctx, &output, &input, ZSTD_e_continue);
     if (ZSTD_isError(ret)) {
       rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
     }
@@ -170,7 +157,7 @@ rb_streaming_compress_write(int argc, VALUE *argv, VALUE obj)
     ZSTD_inBuffer input = { input_data, input_size, 0 };
 
     while (input.pos < input.size) {
-      size_t const ret = ZSTD_compressStream2(sc->ctx, &output, &input, ZSTD_e_continue);
+      size_t const ret = zstd_compress(sc->ctx, &output, &input, ZSTD_e_continue);
       if (ZSTD_isError(ret)) {
         rb_raise(rb_eRuntimeError, "compress error error code: %s", ZSTD_getErrorName(ret));
       }

diff --git a/ext/zstdruby/streaming_compress.h b/ext/zstdruby/streaming_compress.h
diff --git a/ext/zstdruby/streaming_decompress.c b/ext/zstdruby/streaming_decompress.c
@@ -1,7 +1,7 @@
-#include <common.h>
+#include "common.h"
 
 struct streaming_decompress_t {
-  ZSTD_DCtx* ctx;
+  ZSTD_DCtx* dctx;
   VALUE buf;
   size_t buf_size;
 };
@@ -21,9 +21,9 @@ static void
 streaming_decompress_free(void *p)
 {
   struct streaming_decompress_t *sd = p;
-  ZSTD_DCtx* ctx = sd->ctx;
-  if (ctx != NULL) {
-    ZSTD_freeDCtx(ctx);
+  ZSTD_DCtx* dctx = sd->dctx;
+  if (dctx != NULL) {
+    ZSTD_freeDCtx(dctx);
   }
   xfree(sd);
 }
@@ -61,40 +61,29 @@ rb_streaming_decompress_allocate(VALUE klass)
 {
   struct streaming_decompress_t* sd;
   VALUE obj = TypedData_Make_Struct(klass, struct streaming_decompress_t, &streaming_decompress_type, sd);
-  sd->ctx = NULL;
+  sd->dctx = NULL;
   sd->buf = Qnil;
   sd->buf_size = 0;
   return obj;
 }
 
 static VALUE
-rb_streaming_decompress_initialize(VALUE obj)
+rb_streaming_decompress_initialize(int argc, VALUE *argv, VALUE obj)
 {
   VALUE kwargs;
   rb_scan_args(argc, argv, "00:", &kwargs);
 
-  ID kwargs_keys[1];
-  kwargs_keys[0] = rb_intern("dict");
-  VALUE kwargs_values[1];
-  rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
-
   struct streaming_decompress_t* sd;
   TypedData_Get_Struct(obj, struct streaming_decompress_t, &streaming_decompress_type, sd);
   size_t const buffOutSize = ZSTD_DStreamOutSize();
 
-  ZSTD_DCtx* ctx = ZSTD_createDCtx();
-  if (ctx == NULL) {
+  ZSTD_DCtx* dctx = ZSTD_createDCtx();
+  if (dctx == NULL) {
     rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDCtx error");
   }
-  if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
-    char* dict_buffer = RSTRING_PTR(kwargs_values[0]);
-    size_t dict_size = RSTRING_LEN(kwargs_values[0]);
-    size_t load_dict_ret = ZSTD_DCtx_loadDictionary(ctx, dict_buffer, dict_size);
-    if (ZSTD_isError(load_dict_ret)) {
-      rb_raise(rb_eRuntimeError, "%s", "ZSTD_DCtx_loadDictionary failed");
-    }
-  }
-  sd->ctx = ctx;
+  set_decompress_params(dctx, kwargs);
+
+  sd->dctx = dctx;
   sd->buf = rb_str_new(NULL, buffOutSize);
   sd->buf_size = buffOutSize;
 
@@ -115,7 +104,7 @@ rb_streaming_decompress_decompress(VALUE obj, VALUE src)
   VALUE result = rb_str_new(0, 0);
   while (input.pos < input.size) {
     ZSTD_outBuffer output = { (void*)output_data, sd->buf_size, 0 };
-    size_t const ret = ZSTD_decompressStream(sd->ctx, &output, &input);
+    size_t const ret = ZSTD_decompressStream(sd->dctx, &output, &input);
     if (ZSTD_isError(ret)) {
       rb_raise(rb_eRuntimeError, "decompress error error code: %s", ZSTD_getErrorName(ret));
     }
@@ -130,6 +119,6 @@ zstd_ruby_streaming_decompress_init(void)
 {
   VALUE cStreamingDecompress = rb_define_class_under(rb_mZstd, "StreamingDecompress", rb_cObject);
   rb_define_alloc_func(cStreamingDecompress, rb_streaming_decompress_allocate);
-  rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, 0);
+  rb_define_method(cStreamingDecompress, "initialize", rb_streaming_decompress_initialize, -1);
   rb_define_method(cStreamingDecompress, "decompress", rb_streaming_decompress_decompress, 1);
 }