Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Check for the unutilized credit and use them before the pacer period and within the threshold limit #37

Open
wants to merge 1 commit into
base: tuner
Choose a base branch
from
Open

Check for the unutilized credit and use them before the pacer period and within the threshold limit #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions include/spdk/nvmf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ struct spdk_nvmf_io_pacer_stat {
uint64_t calls;
uint64_t no_ios;
uint64_t period_ticks;
uint64_t nos_credit_unavailable;
};

struct spdk_nvmf_transport_poll_group_stat {
Expand Down
85 changes: 66 additions & 19 deletions lib/nvmf/io_pacer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ struct io_pacer_queue {
STAILQ_HEAD(, io_pacer_queue_entry) queue;
};

struct spdk_io_pacer_tuner;
struct spdk_io_pacer_tuner2;
struct spdk_io_pacer {
uint64_t period_ticks;
int64_t credit;
Expand All @@ -72,6 +74,11 @@ struct spdk_io_pacer {
struct io_pacer_queue *queues;
struct spdk_poller *poller;
uint32_t disk_credit;
uint32_t pacer_tuner_type;
union {
struct spdk_io_pacer_tuner *pacer_tuner;
struct spdk_io_pacer_tuner2 *pacer_tuner2;
};
};

struct spdk_io_pacer_tuner {
Expand Down Expand Up @@ -127,22 +134,24 @@ io_pacer_poll(void *arg)
#endif /* SPDK_CONFIG_VTUNE */

pacer->stat.calls++;
if (ticks_diff < pacer->period_ticks) {
return 0;
if (ticks_diff >= pacer->period_ticks){
pacer->stat.total_ticks = cur_tick - pacer->first_tick;
pacer->last_tick = cur_tick - ticks_diff % pacer->period_ticks;
pacer->stat.polls++;
pacer->remaining_credit = spdk_min(pacer->remaining_credit + pacer->credit,
pacer->credit);
if (pacer->num_ios == 0) {
pacer->stat.no_ios++;
}
}
pacer->stat.total_ticks = cur_tick - pacer->first_tick;
pacer->last_tick = cur_tick - ticks_diff % pacer->period_ticks;
pacer->stat.polls++;

pacer->remaining_credit = spdk_min(pacer->remaining_credit + pacer->credit,
pacer->credit);

if (pacer->num_ios == 0) {
pacer->stat.no_ios++;
#if SPDK_NVMF_RDMA_IO_PACER_CHECK_CREDITS_ONLY_AT_PACER_PERIOD
else {
return 0;
}
#endif /* SPDK_NVMF_RDMA_IO_PACER_CHECK_CREDITS_ONLY_AT_PACER_PERIOD */

while ((pacer->num_ios > 0) &&
(pacer->remaining_credit > 0) &&
(is_credit_available(pacer) == true) &&
(attempts_cnt < pacer->num_queues)) {
next_queue %= pacer->num_queues;
attempts_cnt++;
Expand Down Expand Up @@ -182,7 +191,8 @@ struct spdk_io_pacer *
spdk_io_pacer_create(uint32_t period_ns,
uint32_t credit,
uint32_t disk_credit,
spdk_io_pacer_pop_cb pop_cb)
spdk_io_pacer_pop_cb pop_cb,
uint32_t io_pacer_tuner_type)
{
struct spdk_io_pacer *pacer;

Expand All @@ -201,21 +211,23 @@ spdk_io_pacer_create(uint32_t period_ns,
pacer->pop_cb = pop_cb;
pacer->first_tick = spdk_get_ticks();
pacer->last_tick = spdk_get_ticks();
pacer->stat.nos_credit_unavailable = 0;
pacer->poller = SPDK_POLLER_REGISTER(io_pacer_poll, (void *)pacer, 0);
if (!pacer->poller) {
SPDK_ERRLOG("Failed to create poller for IO pacer\n");
spdk_io_pacer_destroy(pacer);
return NULL;
}

SPDK_NOTICELOG("Created IO pacer %p: period_ns %u, period_ticks %lu, max_queues %u, credit %ld, disk_credit %u, core %u\n",
SPDK_NOTICELOG("Created IO pacer %p: period_ns %u, period_ticks %lu, max_queues %u, credit %ld, disk_credit %u, core %u, Tuner type: %u\n",
pacer,
period_ns,
pacer->period_ticks,
pacer->max_queues,
pacer->credit,
pacer->disk_credit,
spdk_env_get_current_core());
spdk_env_get_current_core(),
io_pacer_tuner_type);

return pacer;
}
Expand Down Expand Up @@ -435,6 +447,8 @@ spdk_io_pacer_tuner_create(struct spdk_io_pacer *pacer,
tuner->step_ns = step_ns;
tuner->min_pacer_period_ticks = pacer->period_ticks;
tuner->max_pacer_period_ticks = 2 * tuner->min_pacer_period_ticks;
pacer->pacer_tuner_type = SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01;
pacer->pacer_tuner = tuner;

if (0 != period_us) {
tuner->poller = SPDK_POLLER_REGISTER(io_pacer_tune, (void *)tuner, period_us);
Expand Down Expand Up @@ -475,6 +489,7 @@ struct spdk_io_pacer_tuner2 {
uint64_t min_pacer_period_ticks;
uint64_t max_pacer_period_ticks;
struct spdk_poller *poller;
uint64_t min_threshold_offset;
};

static int
Expand All @@ -492,14 +507,15 @@ io_pacer_tune2(void *arg)
static __thread uint32_t log_counter = 0;
/* Try to log once per second, else it would be too much log */
if (log_counter % (SPDK_SEC_TO_NSEC / tuner->period_ns) == 0) {
SPDK_NOTICELOG("IO pacer tuner %p: pacer %p, value %u, new period %lu ticks, min %lu, polls %u. ios %u\n",
SPDK_NOTICELOG("IO pacer tuner %p: pacer %p, value %u, new period %lu ticks, min %lu, polls %lu. ios %lu, no credit %lu",
tuner,
pacer,
v,
new_period_ticks,
tuner->min_pacer_period_ticks,
pacer->stat.polls,
pacer->stat.ios);
pacer->stat.ios,
pacer->stat.nos_credit_unavailable);
}
log_counter++;

Expand All @@ -515,6 +531,7 @@ spdk_io_pacer_tuner2_create(struct spdk_io_pacer *pacer,
uint64_t factor)
{
struct spdk_io_pacer_tuner2 *tuner;
uint32_t min_threshold_margin;

assert(pacer != NULL);

Expand All @@ -531,7 +548,11 @@ spdk_io_pacer_tuner2_create(struct spdk_io_pacer *pacer,
tuner->factor = factor;
tuner->min_pacer_period_ticks = pacer->period_ticks;
tuner->max_pacer_period_ticks = 4 * tuner->min_pacer_period_ticks;
pacer->pacer_tuner_type = SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02;
pacer->pacer_tuner2 = tuner;

min_threshold_margin = 100 / SPDK_NVMF_RDMA_IO_PACER_DEFAULT_MARGIN_ABOVE_CREDIT; /* variable parameter */
tuner->min_threshold_offset = (tuner->min_threshold / min_threshold_margin);
if (0 != period_us) {
tuner->poller = SPDK_POLLER_REGISTER(io_pacer_tune2, (void *)tuner, period_us);
if (!tuner->poller) {
Expand All @@ -541,12 +562,13 @@ spdk_io_pacer_tuner2_create(struct spdk_io_pacer *pacer,
}
}

SPDK_NOTICELOG("Created IO pacer tuner %p: pacer %p, period_ns %lu, threshold %u, factor %lu\n",
SPDK_NOTICELOG("Created IO pacer tuner %p: pacer %p, period_ns %lu, threshold %u, factor %lu, allowed offset %lu\n",
tuner,
pacer,
tuner->period_ns,
tuner->min_threshold,
tuner->factor);
tuner->factor,
tuner->min_threshold_offset);

return tuner;
}
Expand Down Expand Up @@ -574,3 +596,28 @@ spdk_io_pacer_tuner2_sub(struct spdk_io_pacer_tuner2 *tuner, uint32_t value)
assert(tuner != NULL);
tuner->value -= value;
}

bool
is_credit_available(struct spdk_io_pacer *pacer){
#if SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY
/*
* Check the bytes in fight within allowed offset above the minimum allowed
* threshold value
*/
if (pacer->pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02){
struct spdk_io_pacer_tuner2 *pacer_tuner = pacer->pacer_tuner2;
if ((pacer->remaining_credit > 0)&&
(pacer_tuner->value <= pacer_tuner->min_threshold + pacer_tuner->min_threshold_offset)){
return true;
}
} else if (pacer->remaining_credit > 0){
return true;
}
#else /* SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY */
else if (pacer->remaining_credit > 0){
return true;
}
#endif /* SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY */
pacer->stat.nos_credit_unavailable++;
return false;
}
18 changes: 17 additions & 1 deletion lib/nvmf/io_pacer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,19 @@
#include "spdk_internal/log.h"
#include "spdk/nvmf.h"

/* Tuner types
*/
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01 01
#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02 02

/*
* Configures credit availability to be calculated along with the
* bytes in flight within in the allowed margin offset
*/
#define SPDK_NVMF_RDMA_IO_PACER_CHECK_CREDITS_ONLY_AT_PACER_PERIOD 1
#define SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY 1
#define SPDK_NVMF_RDMA_IO_PACER_DEFAULT_MARGIN_ABOVE_CREDIT 1

struct spdk_io_pacer;
struct spdk_io_pacer_tuner;
struct spdk_io_pacer_tuner2;
Expand All @@ -68,7 +81,8 @@ typedef void (*spdk_io_pacer_pop_cb)(void *io);
struct spdk_io_pacer *spdk_io_pacer_create(uint32_t period_ns,
uint32_t credit,
uint32_t disk_credit,
spdk_io_pacer_pop_cb pop_cb);
spdk_io_pacer_pop_cb pop_cb,
uint32_t io_pacer_tuner_type);
void spdk_io_pacer_destroy(struct spdk_io_pacer *pacer);
int spdk_io_pacer_create_queue(struct spdk_io_pacer *pacer, uint64_t key);
int spdk_io_pacer_destroy_queue(struct spdk_io_pacer *pacer, uint64_t key);
Expand All @@ -89,6 +103,8 @@ void spdk_io_pacer_tuner2_destroy(struct spdk_io_pacer_tuner2 *tuner);
void spdk_io_pacer_tuner2_add(struct spdk_io_pacer_tuner2 *tuner, uint32_t value);
void spdk_io_pacer_tuner2_sub(struct spdk_io_pacer_tuner2 *tuner, uint32_t value);

bool is_credit_available(struct spdk_io_pacer *pacer);

static inline void drive_stats_lock(struct spdk_io_pacer_drives_stats *stats) {
rte_spinlock_lock(&stats->lock);
}
Expand Down
13 changes: 7 additions & 6 deletions lib/nvmf/rdma.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2227,7 +2227,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport,
STAILQ_REMOVE_HEAD(&rgroup->group.pending_buf_queue, buf_link);


if (rgroup->pacer_tuner2 && (rtransport->transport.opts.io_pacer_tuner_type == 1)) {
if (rgroup->pacer_tuner2 && (rtransport->transport.opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02)) {
spdk_io_pacer_tuner2_add(rgroup->pacer_tuner2, rdma_req->req.length);
}

Expand Down Expand Up @@ -2406,7 +2406,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport,
}

if (rgroup->pacer_tuner2 &&
(rtransport->transport.opts.io_pacer_tuner_type == 1)) {
(rtransport->transport.opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02)) {
spdk_io_pacer_tuner2_sub(rgroup->pacer_tuner2, rdma_req->req.length);
}

Expand Down Expand Up @@ -2446,7 +2446,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport,
#define SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP false
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_PERIOD 0
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_THRESHOLD 0
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_TYPE 1 /* Buffers based tuner */
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_TYPE SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02 /* Buffers based tuner */
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_PERIOD 10000 /* us */
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_STEP 1000 /* ns */
#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_THRESHOLD 12*1024*1024
Expand Down Expand Up @@ -3561,15 +3561,16 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport)
rgroup->pacer = spdk_io_pacer_create(transport->opts.io_pacer_period,
transport->opts.io_pacer_credit,
transport->opts.io_pacer_disk_credit,
nvmf_rdma_io_pacer_pop_cb);
nvmf_rdma_io_pacer_pop_cb,
transport->opts.io_pacer_tuner_type);
if (!rgroup->pacer) {
SPDK_ERRLOG("Failed to create IO pacer\n");
spdk_nvmf_rdma_poll_group_destroy(&rgroup->group);
pthread_mutex_unlock(&rtransport->lock);
return NULL;
}

if (transport->opts.io_pacer_tuner_type == 0) {
if (transport->opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01) {
rgroup->pacer_tuner = spdk_io_pacer_tuner_create(rgroup->pacer,
transport->opts.io_pacer_tuner_period,
transport->opts.io_pacer_tuner_step);
Expand Down Expand Up @@ -3718,7 +3719,7 @@ spdk_nvmf_rdma_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group)
rtransport = SPDK_CONTAINEROF(rgroup->group.transport, struct spdk_nvmf_rdma_transport, transport);

if (rgroup->pacer) {
if (rtransport->transport.opts.io_pacer_tuner_type == 0) {
if (rtransport->transport.opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01) {
spdk_io_pacer_tuner_destroy(rgroup->pacer_tuner);
} else {
spdk_io_pacer_tuner2_destroy(rgroup->pacer_tuner2);
Expand Down