Groonga - fulltext search engine.

[Groonga-commit] groonga/groonga at a9cee66 [master] Support Zstandard < 1 again

Kouhei Sutou null+****@clear*****
Thu Apr 5 09:26:38 JST 2018

• Previous message: [Groonga-commit] Build completed: groonga 5930
• Next message: [Groonga-commit] groonga/groonga at d112ea4 [master] package ubuntu: backport Zstandard < 1 support patch for Xenial
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

Kouhei Sutou	2018-04-05 09:26:38 +0900 (Thu, 05 Apr 2018)

  New Revision: a9cee6636b2e3859fcffb04a6ed5a87ac5aef449
  https://github.com/groonga/groonga/commit/a9cee6636b2e3859fcffb04a6ed5a87ac5aef449

  Message:
    Support Zstandard < 1 again

  Modified files:
    lib/store.c

  Modified: lib/store.c (+156 -93)
===================================================================
--- lib/store.c    2018-04-04 09:48:57 +0900 (4fe6444ca)
+++ lib/store.c    2018-04-05 09:26:38 +0900 (dd4b739ef)
@@ -2156,89 +2156,112 @@ grn_ja_putv_zstd(grn_ctx *ctx,
   const size_t body_size = body ? GRN_BULK_VSIZE(body) : 0;
   const size_t footer_size = GRN_BULK_VSIZE(footer);
   const size_t size = header_size + body_size + footer_size;
+  int zstd_compression_level = 3;
+#if ZSTD_VERSION_MAJOR < 1
+  grn_obj all_value;
+  void *zstd_value = NULL;
+
+  GRN_TEXT_INIT(&all_value, 0);
+#else /* ZSTD_VERSION_MAJOR < 1 */
   ZSTD_CStream *zstd_stream = NULL;
-  ZSTD_inBuffer zstd_input;
   ZSTD_outBuffer zstd_output;
-  int zstd_compression_level = 3;
-  size_t zstd_result;
+
+  zstd_output.dst = NULL;
+  zstd_output.pos = 0;
+#endif /* ZSTD_VERSION_MAJOR < 1 */
 
   if (size < COMPRESS_THRESHOLD_BYTE) {
     return grn_ja_putv_packed(ctx, ja, id, header, body, footer, flags);
   }
 
-  zstd_output.dst = NULL;
-  zstd_output.pos = 0;
+#if ZSTD_VERSION_MAJOR < 1
+  {
+    int zstd_value_len_max;
+    int zstd_value_len_real;
 
-  zstd_stream = ZSTD_createCStream();
-  if (!zstd_stream) {
-    grn_ja_compress_error(ctx,
-                          ja,
-                          id,
-                          GRN_ZSTD_ERROR,
-                          "[zstd] failed to allocate stream compressor",
-                          NULL);
-    rc = ctx->rc;
-    goto exit;
-  }
+    zstd_value_len_max = ZSTD_compressBound(size);
+    zstd_value = GRN_MALLOC(zstd_value_len_max);
+    if (!zstd_value) {
+      grn_ja_compress_error(ctx,
+                            ja,
+                            id,
+                            GRN_ZSTD_ERROR,
+                            "[zstd] failed to allocate compress buffer",
+                            NULL);
+      rc = ctx->rc;
+      goto exit;
+    }
 
-  zstd_result = ZSTD_initCStream(zstd_stream, zstd_compression_level);
-  if (ZSTD_isError(zstd_result)) {
-    grn_ja_compress_error(ctx,
-                          ja,
-                          id,
-                          GRN_ZSTD_ERROR,
-                          "[zstd] failed to initialize stream compressor",
-                          ZSTD_getErrorName(zstd_result));
-    rc = ctx->rc;
-    goto exit;
+    GRN_TEXT_PUT(ctx, &all_value, GRN_TEXT_VALUE(header), header_size);
+    if (body_size > 0) {
+      GRN_TEXT_PUT(ctx, &all_value, GRN_TEXT_VALUE(body), body_size);
+    }
+    GRN_TEXT_PUT(ctx, &all_value, GRN_TEXT_VALUE(footer), footer_size);
+    zstd_value_len_real = ZSTD_compress(zstd_value, zstd_value_len_max,
+                                        GRN_TEXT_VALUE(&all_value), size,
+                                        zstd_compression_level);
+    if (ZSTD_isError(zstd_value_len_real)) {
+      grn_ja_compress_error(ctx,
+                            ja,
+                            id,
+                            GRN_ZSTD_ERROR,
+                            "[zstd] failed to compress",
+                            ZSTD_getErrorName(zstd_value_len_real));
+      rc = ctx->rc;
+      goto exit;
+    }
+    rc = grn_ja_putv_compressed(ctx,
+                                ja,
+                                id,
+                                zstd_value,
+                                zstd_value_len_real,
+                                size,
+                                flags);
   }
+#else /* ZSTD_VERSION_MAJOR < 1 */
+  {
+    ZSTD_inBuffer zstd_input;
+    size_t zstd_result;
 
-  zstd_output.size = ZSTD_compressBound(size);
-  zstd_output.dst = GRN_MALLOC(zstd_output.size);
-  if (!zstd_output.dst) {
-    grn_ja_compress_error(ctx,
-                          ja,
-                          id,
-                          GRN_ZSTD_ERROR,
-                          "[zstd] failed to allocate compress buffer",
-                          NULL);
-    rc = ctx->rc;
-    goto exit;
-  }
+    zstd_stream = ZSTD_createCStream();
+    if (!zstd_stream) {
+      grn_ja_compress_error(ctx,
+                            ja,
+                            id,
+                            GRN_ZSTD_ERROR,
+                            "[zstd] failed to allocate stream compressor",
+                            NULL);
+      rc = ctx->rc;
+      goto exit;
+    }
 
-  zstd_input.src = GRN_BULK_HEAD(header);
-  zstd_input.size = header_size;
-  zstd_input.pos = 0;
-  zstd_result = ZSTD_compressStream(zstd_stream, &zstd_output, &zstd_input);
-  if (ZSTD_isError(zstd_result)) {
-    grn_ja_compress_error(ctx,
-                          ja,
-                          id,
-                          GRN_ZSTD_ERROR,
-                          "[zstd] failed to compress header",
-                          ZSTD_getErrorName(zstd_result));
-    rc = ctx->rc;
-    goto exit;
-  }
-  if (body_size > 0) {
-    zstd_input.src = GRN_BULK_HEAD(body);
-    zstd_input.size = body_size;
-    zstd_input.pos = 0;
-    zstd_result = ZSTD_compressStream(zstd_stream, &zstd_output, &zstd_input);
+    zstd_result = ZSTD_initCStream(zstd_stream, zstd_compression_level);
     if (ZSTD_isError(zstd_result)) {
       grn_ja_compress_error(ctx,
                             ja,
                             id,
                             GRN_ZSTD_ERROR,
-                            "[zstd] failed to compress body",
+                            "[zstd] failed to initialize stream compressor",
                             ZSTD_getErrorName(zstd_result));
       rc = ctx->rc;
       goto exit;
     }
-  }
-  if (footer_size > 0) {
-    zstd_input.src = GRN_BULK_HEAD(footer);
-    zstd_input.size = footer_size;
+
+    zstd_output.size = ZSTD_compressBound(size);
+    zstd_output.dst = GRN_MALLOC(zstd_output.size);
+    if (!zstd_output.dst) {
+      grn_ja_compress_error(ctx,
+                            ja,
+                            id,
+                            GRN_ZSTD_ERROR,
+                            "[zstd] failed to allocate compress buffer",
+                            NULL);
+      rc = ctx->rc;
+      goto exit;
+    }
+
+    zstd_input.src = GRN_BULK_HEAD(header);
+    zstd_input.size = header_size;
     zstd_input.pos = 0;
     zstd_result = ZSTD_compressStream(zstd_stream, &zstd_output, &zstd_input);
     if (ZSTD_isError(zstd_result)) {
@@ -2246,52 +2269,92 @@ grn_ja_putv_zstd(grn_ctx *ctx,
                             ja,
                             id,
                             GRN_ZSTD_ERROR,
-                            "[zstd] failed to compress footer",
+                            "[zstd] failed to compress header",
                             ZSTD_getErrorName(zstd_result));
       rc = ctx->rc;
       goto exit;
     }
-  }
+    if (body_size > 0) {
+      zstd_input.src = GRN_BULK_HEAD(body);
+      zstd_input.size = body_size;
+      zstd_input.pos = 0;
+      zstd_result = ZSTD_compressStream(zstd_stream, &zstd_output, &zstd_input);
+      if (ZSTD_isError(zstd_result)) {
+        grn_ja_compress_error(ctx,
+                              ja,
+                              id,
+                              GRN_ZSTD_ERROR,
+                              "[zstd] failed to compress body",
+                              ZSTD_getErrorName(zstd_result));
+        rc = ctx->rc;
+        goto exit;
+      }
+    }
+    if (footer_size > 0) {
+      zstd_input.src = GRN_BULK_HEAD(footer);
+      zstd_input.size = footer_size;
+      zstd_input.pos = 0;
+      zstd_result = ZSTD_compressStream(zstd_stream, &zstd_output, &zstd_input);
+      if (ZSTD_isError(zstd_result)) {
+        grn_ja_compress_error(ctx,
+                              ja,
+                              id,
+                              GRN_ZSTD_ERROR,
+                              "[zstd] failed to compress footer",
+                              ZSTD_getErrorName(zstd_result));
+        rc = ctx->rc;
+        goto exit;
+      }
+    }
 
-  zstd_result = ZSTD_endStream(zstd_stream, &zstd_output);
-  if (ZSTD_isError(zstd_result)) {
-    grn_ja_compress_error(ctx,
-                          ja,
-                          id,
-                          GRN_ZSTD_ERROR,
-                          "[zstd] failed to finish stream compression",
-                          ZSTD_getErrorName(zstd_result));
-    rc = ctx->rc;
-    goto exit;
-  }
+    zstd_result = ZSTD_endStream(zstd_stream, &zstd_output);
+    if (ZSTD_isError(zstd_result)) {
+      grn_ja_compress_error(ctx,
+                            ja,
+                            id,
+                            GRN_ZSTD_ERROR,
+                            "[zstd] failed to finish stream compression",
+                            ZSTD_getErrorName(zstd_result));
+      rc = ctx->rc;
+      goto exit;
+    }
 
-  if (zstd_result > 0) {
-    grn_ja_compress_error(ctx,
-                          ja,
-                          id,
-                          GRN_ZSTD_ERROR,
-                          "[zstd] failed to finish stream compression "
-                          "because some data remain buffer",
-                          ZSTD_getErrorName(zstd_result));
-    rc = ctx->rc;
-    goto exit;
-  }
+    if (zstd_result > 0) {
+      grn_ja_compress_error(ctx,
+                            ja,
+                            id,
+                            GRN_ZSTD_ERROR,
+                            "[zstd] failed to finish stream compression "
+                            "because some data remain buffer",
+                            ZSTD_getErrorName(zstd_result));
+      rc = ctx->rc;
+      goto exit;
+    }
 
-  rc = grn_ja_putv_compressed(ctx,
-                              ja,
-                              id,
-                              zstd_output.dst,
-                              zstd_output.pos,
-                              size,
-                              flags);
+    rc = grn_ja_putv_compressed(ctx,
+                                ja,
+                                id,
+                                zstd_output.dst,
+                                zstd_output.pos,
+                                size,
+                                flags);
+  }
+#endif /* ZSTD_VERSION_MAJOR < 1 */
 
 exit :
+#if ZSTD_VERSION_MAJOR < 1
+  GRN_OBJ_FIN(ctx, &all_value);
+  if (zstd_value) {
+    GRN_FREE(zstd_value);
+  }
+#else /* ZSTD_VERSION_MAJOR < 1 */
   if (zstd_stream) {
     ZSTD_freeCStream(zstd_stream);
   }
   if (zstd_output.dst) {
     GRN_FREE(zstd_output.dst);
   }
+#endif /* ZSTD_VERSION_MAJOR < 1 */
 
   return rc;
 }
-------------- next part --------------
HTML����������������������������...
URL: https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20180405/6267300a/attachment-0001.htm 

• Previous message: [Groonga-commit] Build completed: groonga 5930
• Next message: [Groonga-commit] groonga/groonga at d112ea4 [master] package ubuntu: backport Zstandard < 1 support patch for Xenial
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]