Skip to content

Commit

Permalink
refactoring of Cntd struct
Browse files Browse the repository at this point in the history
  • Loading branch information
@cpt-harlock
cpt-harlock committed Jan 9, 2025
1 parent 3039458 commit 93c9fe3
Show file tree
Hide file tree
Showing 7 changed files with 98 additions and 103 deletions.
16 changes: 8 additions & 8 deletions src/arch.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ HIDDEN void init_perf()
gethostname(hostname, sizeof(hostname));
PMPI_Comm_rank(MPI_COMM_WORLD, &world_rank);

for (i = 0; i < cntd->local_rank_size; i++) {
for (i = 0; i < cntd->rank->local_size; i++) {
pid = cntd->local_ranks[i]->pid;

memset(&perf_pe, 0, sizeof(perf_pe));
Expand Down Expand Up @@ -385,9 +385,9 @@ HIDDEN void init_perf()
}
}

PMPI_Barrier(cntd->comm_local_masters);
PMPI_Barrier(cntd->comm_masters);

for (i = 0; i < cntd->local_rank_size; i++) {
for (i = 0; i < cntd->rank->local_size; i++) {
ioctl(cntd->perf_fd[i][PERF_INST_RET], PERF_EVENT_IOC_ENABLE,
0);
ioctl(cntd->perf_fd[i][PERF_CYCLES], PERF_EVENT_IOC_ENABLE, 0);
Expand Down Expand Up @@ -637,9 +637,9 @@ HIDDEN void finalize_perf()
{
int i, j;

PMPI_Barrier(cntd->comm_local_masters);
PMPI_Barrier(cntd->comm_masters);

for (i = 0; i < cntd->local_rank_size; i++) {
for (i = 0; i < cntd->rank->local_size; i++) {
ioctl(cntd->perf_fd[i][PERF_INST_RET], PERF_EVENT_IOC_DISABLE,
0);
ioctl(cntd->perf_fd[i][PERF_CYCLES], PERF_EVENT_IOC_DISABLE, 0);
Expand All @@ -658,7 +658,7 @@ HIDDEN void finalize_perf()
}
}

for (i = 0; i < cntd->local_rank_size; i++) {
for (i = 0; i < cntd->rank->local_size; i++) {
close(cntd->perf_fd[i][PERF_INST_RET]);
close(cntd->perf_fd[i][PERF_CYCLES]);
#ifdef INTEL
Expand All @@ -678,7 +678,7 @@ HIDDEN void init_arch_conf()
{
hwloc_topology_t topology;
int i, depth, world_rank, pid;
int pids[cntd->local_rank_size];
int pids[cntd->rank->local_size];
char hostname[STRING_SIZE];

gethostname(hostname, sizeof(hostname));
Expand Down Expand Up @@ -746,7 +746,7 @@ HIDDEN void init_arch_conf()
pid = getpid();
PMPI_Gather(&pid, 1, MPI_INT, pids, 1, MPI_INT, 0, cntd->comm_local);
if (cntd->rank->local_rank == 0) {
for (i = 0; i < cntd->local_rank_size; i++)
for (i = 0; i < cntd->rank->local_size; i++)
cntd->local_ranks[i]->pid = pids[i];
}
}
Expand Down
26 changes: 13 additions & 13 deletions src/cntd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,14 @@
*/

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
#include <time.h>
#include <dirent.h>
#include <unistd.h>
#include <sched.h>
#include <signal.h>
#include <stdint.h>
#include <string.h>
#include <fcntl.h>
#include <errno.h>
#include <sys/resource.h>
#include <sys/mman.h>
#include <sys/sysinfo.h>
Expand Down Expand Up @@ -92,7 +88,7 @@
#define DEFAULT_SAMPLING_TIME_REPORT 1 // 1 second
#define MAX_NUM_SOCKETS 16 // Max supported sockets in a single node
#define MAX_NUM_GPUS 16 // Max supported gpus in a single node
#define MAX_NUM_CPUS 1024 // Max supported CPUS in a single node
#define MAX_NUM_CPUS 8192 // Max supported CPUS in a single node

// EAM configurations
#define DEFAULT_TIMEOUT 0.0005 // 500us
Expand Down Expand Up @@ -450,12 +446,13 @@ typedef struct {
int world_rank;
int local_rank;

int world_size;
int local_size;

char hostname[STRING_SIZE];
int cpu_id;
int pid;

int exe_is_started;

uint64_t num_sampling;

double exe_time[2];
Expand Down Expand Up @@ -523,17 +520,21 @@ typedef struct {
char tmp_dir[STRING_SIZE];

unsigned int force_msr : 1;
unsigned int enable_cntd : 1;
unsigned int enable_cntd_slack : 1;
unsigned int enable_eam : 1;
unsigned int enable_eam_slack : 1;
unsigned int enable_eam_freq : 1;
unsigned int enable_power_monitor : 1;
unsigned int enable_timeseries_report : 1;
unsigned int enable_report : 1;
unsigned int enable_perf : 1;

// MPI Communicators
MPI_Comm comm_local;
MPI_Comm comm_local_masters;
int local_rank_size;
MPI_Comm comm_masters;
// MPI Ranks
// World rank of node master
int master_rank;
int iam_master;

unsigned int into_mpi : 1;

Expand All @@ -559,6 +560,7 @@ typedef struct {

int nom_freq_mhz;
#ifdef INTEL
_Bool hwp_usage;
int msr_fd;
int energy_pkg_fd[MAX_NUM_SOCKETS];
double energy_pkg_overflow[MAX_NUM_SOCKETS];
Expand All @@ -576,8 +578,6 @@ typedef struct {

extern CNTD_t *cntd;

extern _Bool hwp_usage;

#ifdef MOSQUITTO_ENABLED
typedef struct mosquitto MOSQUITTO_t;

Expand Down
2 changes: 0 additions & 2 deletions src/hwp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@

#include "cntd.h"

_Bool hwp_usage;

#ifdef HWP_AVAIL
// Max "Energy_Performance_Preference".
HIDDEN void set_max_epp()
Expand Down
64 changes: 31 additions & 33 deletions src/init.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,12 @@ static void read_env()
char *cntd_enable = getenv("CNTD_ENABLE");
if (cntd_enable != NULL) {
if (strcasecmp(cntd_enable, "analysis") == 0) {
cntd->enable_cntd = TRUE;
cntd->enable_cntd_slack = FALSE;
cntd->enable_eam = TRUE;
cntd->enable_eam_slack = FALSE;
cntd->enable_eam_freq = FALSE;
} else if (str_to_bool(cntd_enable)) {
cntd->enable_cntd = TRUE;
cntd->enable_cntd_slack = FALSE;
cntd->enable_eam = TRUE;
cntd->enable_eam_slack = FALSE;
cntd->enable_eam_freq = TRUE;
} else {
fprintf(stderr,
Expand All @@ -66,12 +66,12 @@ static void read_env()
char *cntd_slack_enable_str = getenv("CNTD_SLACK_ENABLE");
if (cntd_slack_enable_str != NULL) {
if (strcasecmp(cntd_slack_enable_str, "analysis") == 0) {
cntd->enable_cntd = FALSE;
cntd->enable_cntd_slack = TRUE;
cntd->enable_eam = FALSE;
cntd->enable_eam_slack = TRUE;
cntd->enable_eam_freq = FALSE;
} else if (str_to_bool(cntd_slack_enable_str)) {
cntd->enable_cntd = FALSE;
cntd->enable_cntd_slack = TRUE;
cntd->enable_eam = FALSE;
cntd->enable_eam_slack = TRUE;
cntd->enable_eam_freq = TRUE;
} else {
fprintf(stderr,
Expand Down Expand Up @@ -157,11 +157,11 @@ static void read_env()
snprintf(perf_env, sizeof(perf_env), "CNTD_PERF_EVENT_%d", j);
char *cntd_perf_event = getenv(perf_env);
if (cntd_perf_event != NULL)
for (i = 0; i < cntd->local_rank_size; i++)
for (i = 0; i < cntd->rank->local_size; i++)
cntd->perf_fd[i][j] =
(int)strtoul(cntd_perf_event, 0L, 16);
else
for (i = 0; i < cntd->local_rank_size; i++)
for (i = 0; i < cntd->rank->local_size; i++)
cntd->perf_fd[i][j] = 0;
}

Expand Down Expand Up @@ -218,7 +218,6 @@ static void read_env()
static void init_local_masters()
{
int i, local_master;
int iam_local_master;
int world_rank, local_rank, world_size;
char hostname[STRING_SIZE];
char postfix[STRING_SIZE], shmem_name[STRING_SIZE];
Expand All @@ -236,34 +235,35 @@ static void init_local_masters()
// Find local master and local communicators
for (i = 0; i < world_size; i++) {
if (strncmp(hostname, global_hostname[i], STRING_SIZE) == 0) {
local_master = i;
cntd->master_rank = i;
break;
}
}
if (world_rank == local_master)
iam_local_master = TRUE;
else
iam_local_master = FALSE;
PMPI_Comm_split(MPI_COMM_WORLD, iam_local_master, 0,
&cntd->comm_local_masters);
PMPI_Comm_split(MPI_COMM_WORLD, local_master, 0, &cntd->comm_local);
if (world_rank == cntd->master_rank)
cntd->iam_master = TRUE;
// Create local masters' group communicator aka group of masters
PMPI_Comm_split(MPI_COMM_WORLD, cntd->iam_master, 0,
&cntd->comm_masters);
// Create node communicator aka master + slaves
PMPI_Comm_split(MPI_COMM_WORLD, cntd->master_rank, 0,
&cntd->comm_local);
PMPI_Comm_rank(cntd->comm_local, &local_rank);

// Init shared memory
get_rand_postfix(postfix, STRING_SIZE);
snprintf(shmem_name, sizeof(shmem_name), SHM_FILE, local_rank, postfix);
cntd->local_ranks[local_rank] = create_shmem_rank(shmem_name, 1);
cntd->rank = cntd->local_ranks[local_rank];
cntd->rank->exe_is_started = 0;

PMPI_Comm_size(cntd->comm_local, &cntd->local_rank_size);
PMPI_Comm_size(cntd->comm_local, &cntd->rank->local_size);
cntd->rank->world_size = world_size;

PMPI_Barrier(MPI_COMM_WORLD);

cntd->rank->world_rank = world_rank;
cntd->rank->local_rank = local_rank;

for (i = 0; i < cntd->local_rank_size; i++) {
for (i = 0; i < cntd->rank->local_size; i++) {
if (i == local_rank)
continue;
else {
Expand All @@ -288,8 +288,6 @@ HIDDEN void start_cntd()
{
cntd = (CNTD_t *)calloc(1, sizeof(CNTD_t));

hwp_usage = 0;

// Init local masters
init_local_masters();

Expand All @@ -306,7 +304,7 @@ HIDDEN void start_cntd()
pstate = read_msr(IA32_PM_ENABLE);

if (pstate)
hwp_usage = 1;
cntd->hwp_usage = 1;
else
fprintf(stdout,
"Warning: HWP-States available, but not usable.\n");
Expand Down Expand Up @@ -341,18 +339,18 @@ HIDDEN void start_cntd()
init_timeseries_report();

// Init energy-aware MPI
if (cntd->enable_cntd)
if (cntd->enable_eam)
eam_init();
else if (cntd->enable_cntd_slack)
else if (cntd->enable_eam_slack)
eam_slack_init();
}

HIDDEN void stop_cntd()
{
// Finalize energy-aware MPI
if (cntd->enable_cntd)
if (cntd->enable_eam)
eam_finalize();
else if (cntd->enable_cntd_slack)
else if (cntd->enable_eam_slack)
eam_slack_finalize();

finalize_time_sample();
Expand Down Expand Up @@ -404,9 +402,9 @@ HIDDEN void call_start(MPI_Type_t mpi_type, MPI_Comm comm, int addr)
{
cntd->into_mpi = TRUE;

if (cntd->enable_cntd)
if (cntd->enable_eam)
eam_start_mpi();
else if (cntd->enable_cntd_slack)
else if (cntd->enable_eam_slack)
eam_slack_start_mpi(mpi_type, comm, addr);

event_sample_start(mpi_type);
Expand All @@ -417,9 +415,9 @@ HIDDEN void call_end(MPI_Type_t mpi_type, MPI_Comm comm, int addr)
{
int eam_flag = FALSE;

if (cntd->enable_cntd)
if (cntd->enable_eam)
eam_flag = eam_end_mpi();
else if (cntd->enable_cntd_slack)
else if (cntd->enable_eam_slack)
eam_flag = eam_slack_end_mpi(mpi_type, comm, addr);

event_sample_end(mpi_type, eam_flag);
Expand Down
6 changes: 3 additions & 3 deletions src/pm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ HIDDEN void set_pstate(int pstate)
int offset = IA32_PERF_CTL;
written_pstate = (pstate << 8) & 0xFF00;
#ifdef HWP_AVAIL
if (hwp_usage) {
if (cntd->hwp_usage) {
offset = IA32_HWP_REQUEST;
/*
This is needed to write also \"Minimum_Performance\" field of this
Expand All @@ -129,7 +129,7 @@ HIDDEN void set_pstate(int pstate)
HIDDEN void set_max_pstate()
{
//#ifdef HWP_AVAIL
// if (hwp_usage) {
// if (cntd->hwp_usage) {
// set_min_epp();
// set_min_aw();
// }
Expand Down Expand Up @@ -260,7 +260,7 @@ HIDDEN int get_minimum_frequency()
{
if (cntd->enable_eam_freq) {
#ifdef HWP_AVAIL
if (hwp_usage) {
if (cntd->hwp_usage) {
int offset;
int min_pstate;

Expand Down
Loading

0 comments on commit 93c9fe3

Please sign in to comment.