Merge remote-tracking branch 'ceph/master'

ceph.spec.in

@@ -1647,6 +1647,7 @@ fi
 %{_datadir}/ceph/mgr/rbd_support
 %{_datadir}/ceph/mgr/restful
 %{_datadir}/ceph/mgr/selftest
+%{_datadir}/ceph/mgr/snap_schedule
 %{_datadir}/ceph/mgr/status
 %{_datadir}/ceph/mgr/telegraf
 %{_datadir}/ceph/mgr/telemetry

debian/ceph-mgr-modules-core.install

@@ -14,6 +14,7 @@ usr/share/ceph/mgr/prometheus
 usr/share/ceph/mgr/rbd_support
 usr/share/ceph/mgr/restful
 usr/share/ceph/mgr/selftest
+usr/share/ceph/mgr/snap_schedule
 usr/share/ceph/mgr/status
 usr/share/ceph/mgr/telegraf
 usr/share/ceph/mgr/telemetry

doc/radosgw/qat-accel.rst

@@ -38,8 +38,8 @@ The driver package can be downloaded from `Intel Quickassist Technology`_.
 2. The implementation for QAT based encryption is directly base on QAT API which
    is included the driver package. But QAT support for compression depends on
    QATzip project, which is a user space library which builds on top of the QAT
-   API. QATZip can support several compression algorithm, including deflate,
-   snappy, lz4, etc..
+   API. Currently, QATzip speeds up gzip compression and decompression at the
+   time of writing.
 
 See `QATzip`_.
 
@@ -62,10 +62,10 @@ As mentioned above, QAT support for compression is based on QATzip library in
 user space, which is designed to take full advantage of the performance provided
 by QuickAssist Technology. Unlike QAT based encryption, QAT based compression
 is supported through a tool class for QAT acceleration rather than a compressor
-plugin. The common tool class will be shared among zip, snappy, lz4 compressor
-plugins, and can transparently accelerate the existing compression types. So
-user is allowed to use it to speed up the existing compression types as long as
-the QAT hardware is available and QAT is capable to handle them.
+plugin. The common tool class can transparently accelerate the existing compression
+types, but only zlib compressor can be supported at the time of writing. So
+user is allowed to use it to speed up zlib compressor as long as the QAT
+hardware is available and QAT is capable to handle it.
 
 Configuration
 =============

src/blk/kernel/KernelDevice.cc

@@ -85,12 +85,25 @@ KernelDevice::KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, ai
 int KernelDevice::_lock()
 {
   dout(10) << __func__ << " " << fd_directs[WRITE_LIFE_NOT_SET] << dendl;
-  int r = ::flock(fd_directs[WRITE_LIFE_NOT_SET], LOCK_EX | LOCK_NB);
-  if (r < 0) {
-    derr << __func__ << " flock failed on " << path << dendl;
-    return -errno;
+  utime_t sleeptime;
+  sleeptime.set_from_double(cct->_conf->bdev_flock_retry_interval);
+
+  // When the block changes, systemd-udevd will open the block,
+  // read some information and close it. Then a failure occurs here.
+  // So we need to try again here.
+  for (int i = 0; i < cct->_conf->bdev_flock_retry + 1; i++) {
+    int r = ::flock(fd_directs[WRITE_LIFE_NOT_SET], LOCK_EX | LOCK_NB);
+    if (r < 0 && errno == EAGAIN) {
+      dout(1) << __func__ << " flock busy on " << path << dendl;    
+      sleeptime.sleep();
+    } else if (r < 0) {
+      derr << __func__ << " flock failed on " << path << dendl;    
+      break;
+    } else {
+      return 0;
+    }
   }
-  return 0;
+  return -errno;
 }
 
 int KernelDevice::open(const string& p)

src/blk/spdk/NVMEDevice.cc

@@ -922,11 +922,11 @@ int NVMEDevice::read(uint64_t off, uint64_t len, bufferlist *pbl,
   bufferptr p = buffer::create_small_page_aligned(len);
   char *buf = p.c_str();
 
-  ceph_assert(ioc->nvme_task_first == nullptr);
-  ceph_assert(ioc->nvme_task_last == nullptr);
-  make_read_tasks(this, off, ioc, buf, len, &t, off, len);
+  // for sync read, need to control IOContext in itself
+  IOContext read_ioc(cct, nullptr);
+  make_read_tasks(this, off, &read_ioc, buf, len, &t, off, len);
   dout(5) << __func__ << " " << off << "~" << len << dendl;
-  aio_submit(ioc);
+  aio_submit(&read_ioc);
 
   pbl->push_back(std::move(p));
   return t.return_code;

src/common/legacy_config_opts.h

@@ -898,6 +898,8 @@ OPTION(bdev_nvme_unbind_from_kernel, OPT_BOOL)
 OPTION(bdev_nvme_retry_count, OPT_INT) // -1 means by default which is 4
 OPTION(bdev_enable_discard, OPT_BOOL)
 OPTION(bdev_async_discard, OPT_BOOL)
+OPTION(bdev_flock_retry_interval, OPT_FLOAT)
+OPTION(bdev_flock_retry, OPT_INT)
 
 OPTION(objectstore_blackhole, OPT_BOOL)
 

src/common/options.cc

@@ -3992,6 +3992,14 @@ std::vector<Option> get_global_options() {
     Option("bdev_async_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(false)
     .set_description(""),
+
+    Option("bdev_flock_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(0.1)
+    .set_description("interval to retry the flock"),
+
+    Option("bdev_flock_retry", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(3)
+    .set_description("times to retry the flock"),
 
     Option("bluefs_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
     .set_default(1_M)

src/librbd/CMakeLists.txt

@@ -167,28 +167,13 @@ set(librbd_internal_srcs
   operation/SnapshotLimitRequest.cc
   operation/SparsifyRequest.cc
   operation/TrimRequest.cc
+  plugin/Api.cc
   trash/MoveRequest.cc
   trash/RemoveRequest.cc
   watcher/Notifier.cc
   watcher/RewatchRequest.cc
   ${CMAKE_SOURCE_DIR}/src/common/ContextCompletion.cc)
 
-add_custom_target(librbd_plugins)
-set(librbd_plugins_dir ${CEPH_INSTALL_PKGLIBDIR}/librbd)
-
-set(rbd_plugin_parent_cache_srcs
-  cache/ParentCacheObjectDispatch.cc
-  plugin/ParentCache.cc)
-add_library(librbd_plugin_parent_cache SHARED ${rbd_plugin_parent_cache_srcs})
-target_link_libraries(librbd_plugin_parent_cache PRIVATE
-  ceph_immutable_object_cache_lib)
-set_target_properties(librbd_plugin_parent_cache PROPERTIES
-  OUTPUT_NAME ceph_librbd_parent_cache
-  VERSION 1.0.0
-  SOVERSION 1)
-install(TARGETS librbd_plugin_parent_cache DESTINATION ${librbd_plugins_dir})
-add_dependencies(librbd_plugins librbd_plugin_parent_cache)
-
 if(WITH_EVENTTRACE)
   list(APPEND librbd_internal_srcs ../common/EventTrace.cc)
 endif()
@@ -228,6 +213,24 @@ if(WITH_RBD_RWL)
     PUBLIC blk)
 endif()
 
+add_custom_target(librbd_plugins)
+set(librbd_plugins_dir ${CEPH_INSTALL_PKGLIBDIR}/librbd)
+
+set(rbd_plugin_parent_cache_srcs
+  cache/ParentCacheObjectDispatch.cc
+  plugin/ParentCache.cc)
+add_library(librbd_plugin_parent_cache SHARED
+  ${rbd_plugin_parent_cache_srcs})
+target_link_libraries(librbd_plugin_parent_cache PRIVATE
+  ceph_immutable_object_cache_lib
+  librados)
+set_target_properties(librbd_plugin_parent_cache PROPERTIES
+  OUTPUT_NAME ceph_librbd_parent_cache
+  VERSION 1.0.0
+  SOVERSION 1)
+install(TARGETS librbd_plugin_parent_cache DESTINATION ${librbd_plugins_dir})
+add_dependencies(librbd_plugins librbd_plugin_parent_cache)
+
 add_library(librbd ${CEPH_SHARED}
   librbd.cc)
 if(WITH_LTTNG)

src/librbd/PluginRegistry.cc

@@ -5,6 +5,7 @@
 #include "include/Context.h"
 #include "common/dout.h"
 #include "librbd/ImageCtx.h"
+#include "librbd/plugin/Api.h"
 #include <boost/tokenizer.hpp>
 
 #define dout_subsys ceph_subsys_rbd
@@ -15,7 +16,12 @@
 namespace librbd {
 
 template <typename I>
-PluginRegistry<I>::PluginRegistry(I* image_ctx) : m_image_ctx(image_ctx) {
+PluginRegistry<I>::PluginRegistry(I* image_ctx)
+  : m_image_ctx(image_ctx), m_plugin_api(std::make_unique<plugin::Api<I>>()) {
+}
+
+template <typename I>
+PluginRegistry<I>::~PluginRegistry() {
 }
 
 template <typename I>
@@ -35,13 +41,13 @@ void PluginRegistry<I>::init(const std::string& plugins, Context* on_finish) {
       plugin_registry->get_with_load("librbd", "librbd_" + token));
     if (plugin == nullptr) {
       lderr(cct) << "failed to load plugin: " << token << dendl;
-      ctx->complete(-ENOENT);
+      ctx->complete(-ENOSYS);
       break;
     }
 
     m_plugin_hook_points.emplace_back();
     auto hook_points = &m_plugin_hook_points.back();
-    plugin->init(m_image_ctx, hook_points, ctx);
+    plugin->init(m_image_ctx, *m_plugin_api, hook_points, ctx);
   }
 
   gather_ctx->activate();

src/librbd/PluginRegistry.h

@@ -5,6 +5,7 @@
 #define CEPH_LIBRBD_PLUGIN_REGISTRY_H
 
 #include "librbd/plugin/Types.h"
+#include <memory>
 #include <string>
 #include <list>
 
@@ -14,17 +15,22 @@ namespace librbd {
 
 struct ImageCtx;
 
+namespace plugin { template <typename> struct Api; }
+
 template <typename ImageCtxT>
 class PluginRegistry {
 public:
   PluginRegistry(ImageCtxT* image_ctx);
+  ~PluginRegistry();
 
   void init(const std::string& plugins, Context* on_finish);
 
 private:
   typedef std::list<plugin::HookPoints> PluginHookPoints;
 
   ImageCtxT* m_image_ctx;
+  std::unique_ptr<plugin::Api<ImageCtxT>> m_plugin_api;
+
   std::string m_plugins;
 
   PluginHookPoints m_plugin_hook_points;

src/librbd/cache/ParentCacheObjectDispatch.cc

@@ -5,10 +5,10 @@
 #include "librbd/ImageCtx.h"
 #include "librbd/Utils.h"
 #include "librbd/asio/ContextWQ.h"
+#include "librbd/cache/ParentCacheObjectDispatch.h"
 #include "librbd/io/ObjectDispatchSpec.h"
 #include "librbd/io/ObjectDispatcherInterface.h"
-#include "librbd/io/Utils.h"
-#include "librbd/cache/ParentCacheObjectDispatch.h"
+#include "librbd/plugin/Api.h"
 #include "osd/osd_types.h"
 #include "osdc/WritebackHandler.h"
 
@@ -27,8 +27,8 @@ namespace cache {
 
 template <typename I>
 ParentCacheObjectDispatch<I>::ParentCacheObjectDispatch(
-    I* image_ctx)
-  : m_image_ctx(image_ctx),
+    I* image_ctx, plugin::Api<I>& plugin_api)
+  : m_image_ctx(image_ctx), m_plugin_api(plugin_api),
     m_lock(ceph::make_mutex(
       "librbd::cache::ParentCacheObjectDispatch::lock", true, false)) {
   ceph_assert(m_image_ctx->data_ctx.is_valid());
@@ -136,7 +136,7 @@ void ParentCacheObjectDispatch<I>::handle_read_cache(
         *dispatch_result = io::DISPATCH_RESULT_COMPLETE;
         on_dispatched->complete(r);
       });
-    io::util::read_parent<I>(m_image_ctx, object_no, {{read_off, read_len}},
+    m_plugin_api.read_parent(m_image_ctx, object_no, {{read_off, read_len}},
                              snap_id, parent_trace, read_data, ctx);
     return;
   }

src/librbd/cache/ParentCacheObjectDispatch.h

@@ -14,6 +14,8 @@ namespace librbd {
 
 class ImageCtx;
 
+namespace plugin { template <typename> struct Api; }
+
 namespace cache {
 
 template <typename ImageCtxT = ImageCtx>
@@ -23,11 +25,13 @@ class ParentCacheObjectDispatch : public io::ObjectDispatchInterface {
   typedef typename TypeTraits::CacheClient CacheClient;
 
 public:
-  static ParentCacheObjectDispatch* create(ImageCtxT* image_ctx) {
-    return new ParentCacheObjectDispatch(image_ctx);
+  static ParentCacheObjectDispatch* create(ImageCtxT* image_ctx,
+                                           plugin::Api<ImageCtxT>& plugin_api) {
+    return new ParentCacheObjectDispatch(image_ctx, plugin_api);
   }
 
-  ParentCacheObjectDispatch(ImageCtxT* image_ctx);
+  ParentCacheObjectDispatch(ImageCtxT* image_ctx,
+                            plugin::Api<ImageCtxT>& plugin_api);
   ~ParentCacheObjectDispatch() override;
 
   io::ObjectDispatchLayer get_dispatch_layer() const override {
@@ -129,6 +133,7 @@ class ParentCacheObjectDispatch : public io::ObjectDispatchInterface {
   void create_cache_session(Context* on_finish, bool is_reconnect);
 
   ImageCtxT* m_image_ctx;
+  plugin::Api<ImageCtxT>& m_plugin_api;
 
   ceph::mutex m_lock;
   CacheClient *m_cache_client = nullptr;

src/librbd/image/OpenRequest.cc

@@ -549,6 +549,8 @@ Context* OpenRequest<I>::handle_init_plugin_registry(int *result) {
   if (*result < 0) {
     lderr(cct) << "failed to initialize plugin registry: "
                << cpp_strerror(*result) << dendl;
+    send_close_image(*result);
+    return nullptr;
   }
 
   return send_init_cache(result);

src/librbd/plugin/Api.cc

@@ -0,0 +1,23 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/plugin/Api.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/io/Utils.h"
+
+namespace librbd {
+namespace plugin {
+
+template <typename I>
+void Api<I>::read_parent(
+    I *image_ctx, uint64_t object_no, const Extents &extents,
+    librados::snap_t snap_id, const ZTracer::Trace &trace,
+    ceph::bufferlist* data, Context* on_finish) {
+  io::util::read_parent<I>(image_ctx, object_no, extents, snap_id, trace, data,
+                           on_finish);
+}
+
+} // namespace plugin
+} // namespace librbd
+
+template class librbd::plugin::Api<librbd::ImageCtx>;

src/librbd/plugin/Api.h

@@ -0,0 +1,39 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_PLUGIN_API_H
+#define CEPH_LIBRBD_PLUGIN_API_H
+
+#include "include/common_fwd.h"
+#include "include/int_types.h"
+#include "include/rados/librados.hpp"
+#include "librbd/io/Types.h"
+
+namespace ZTracer { struct Trace; }
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace plugin {
+
+template <typename ImageCtxT>
+struct Api {
+  using Extents = librbd::io::Extents;
+
+  Api() {}
+  virtual ~Api() {}
+
+  virtual void read_parent(
+      ImageCtxT *image_ctx, uint64_t object_no, const Extents &extents,
+      librados::snap_t snap_id, const ZTracer::Trace &trace,
+      ceph::bufferlist* data, Context* on_finish);
+
+};
+
+} // namespace plugin
+} // namespace librbd
+
+extern template class librbd::plugin::Api<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_PLUGIN_API_H

src/librbd/plugin/ParentCache.cc

@@ -33,7 +33,7 @@ namespace librbd {
 namespace plugin {
 
 template <typename I>
-void ParentCache<I>::init(I* image_ctx, HookPoints* hook_points,
+void ParentCache<I>::init(I* image_ctx, Api<I>& api, HookPoints* hook_points,
                           Context* on_finish) {
   m_image_ctx = image_ctx;
   bool parent_cache_enabled = m_image_ctx->config.template get_val<bool>(
@@ -46,7 +46,8 @@ void ParentCache<I>::init(I* image_ctx, HookPoints* hook_points,
   auto cct = m_image_ctx->cct;
   ldout(cct, 5) << dendl;
 
-  auto parent_cache = cache::ParentCacheObjectDispatch<I>::create(m_image_ctx);
+  auto parent_cache = cache::ParentCacheObjectDispatch<I>::create(
+    m_image_ctx, api);
   on_finish = new LambdaContext([this, on_finish, parent_cache](int r) {
       if (r < 0) {
         // the object dispatcher will handle cleanup if successfully initialized