diff options
67 files changed, 4373 insertions, 1604 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index e0c114ab4b..a23f80e35d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -731,6 +731,8 @@ set(RRD_PLUGIN_FILES database/rrddimvar.c database/rrddimvar.h database/rrdfamily.c + database/rrdfunctions.c + database/rrdfunctions.h database/rrdhost.c database/rrdlabels.c database/rrd.c diff --git a/Makefile.am b/Makefile.am index 34026e9888..1a18150314 100644 --- a/Makefile.am +++ b/Makefile.am @@ -441,6 +441,8 @@ RRD_PLUGIN_FILES = \ database/rrd.c \ database/rrd.h \ database/rrdset.c \ + database/rrdfunctions.c \ + database/rrdfunctions.h \ database/rrdsetvar.c \ database/rrdsetvar.h \ database/rrdvar.c \ diff --git a/aclk/aclk_query.c b/aclk/aclk_query.c index 132d5fe18f..b30da60e1f 100644 --- a/aclk/aclk_query.c +++ b/aclk/aclk_query.c @@ -80,7 +80,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query) strcpy(w->origin, "*"); // Simulate web_client_create_on_fd() w->cookie1[0] = 0; // Simulate web_client_create_on_fd() w->cookie2[0] = 0; // Simulate web_client_create_on_fd() - w->acl = 0x1f; + w->acl = WEB_CLIENT_ACL_ACLK; buffer_strcat(log_buffer, query->data.http_api_v2.query); size_t size = 0; diff --git a/claim/claim.c b/claim/claim.c index b0d40ecf86..c9b1712f86 100644 --- a/claim/claim.c +++ b/claim/claim.c @@ -58,7 +58,7 @@ void claim_agent(char *claiming_arguments) int exit_code; pid_t command_pid; char command_buffer[CLAIMING_COMMAND_LENGTH + 1]; - FILE *fp; + FILE *fp_child_output, *fp_child_input; // This is guaranteed to be set early in main via post_conf_load() char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL); @@ -84,14 +84,14 @@ void claim_agent(char *claiming_arguments) claiming_arguments); info("Executing agent claiming command 'netdata-claim.sh'"); - fp = mypopen(command_buffer, &command_pid); - if(!fp) { + fp_child_output = netdata_popen(command_buffer, &command_pid, &fp_child_input); + if(!fp_child_output) { error("Cannot popen(\"%s\").", command_buffer); return; } info("Waiting for claiming command to finish."); - while (fgets(command_buffer, CLAIMING_COMMAND_LENGTH, fp) != NULL) {;} - exit_code = mypclose(fp, command_pid); + while (fgets(command_buffer, CLAIMING_COMMAND_LENGTH, fp_child_output) != NULL) {;} + exit_code = netdata_pclose(fp_child_input, fp_child_output, command_pid); info("Agent claiming command returned with code %d", exit_code); if (0 == exit_code) { load_claiming_state(); diff --git a/collectors/apps.plugin/apps_plugin.c b/collectors/apps.plugin/apps_plugin.c index 8521e078e8..212374e828 100644 --- a/collectors/apps.plugin/apps_plugin.c +++ b/collectors/apps.plugin/apps_plugin.c @@ -10,6 +10,11 @@ #include "libnetdata/libnetdata.h" #include "libnetdata/required_dummies.h" +#define APPS_PLUGIN_FUNCTIONS() do { \ + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" 10 \"Detailed information on the currently running processes on this node\"\n"); \ + } while(0) + + // ---------------------------------------------------------------------------- // debugging @@ -191,6 +196,18 @@ struct pid_on_target { struct pid_on_target *next; }; +struct openfds { + kernel_uint_t files; + kernel_uint_t pipes; + kernel_uint_t sockets; + kernel_uint_t inotifies; + kernel_uint_t eventfds; + kernel_uint_t timerfds; + kernel_uint_t signalfds; + kernel_uint_t eventpolls; + kernel_uint_t other; +}; + // ---------------------------------------------------------------------------- // target // @@ -235,24 +252,16 @@ struct target { kernel_uint_t io_logical_bytes_read; kernel_uint_t io_logical_bytes_written; - // kernel_uint_t io_read_calls; - // kernel_uint_t io_write_calls; + kernel_uint_t io_read_calls; + kernel_uint_t io_write_calls; kernel_uint_t io_storage_bytes_read; kernel_uint_t io_storage_bytes_written; - // kernel_uint_t io_cancelled_write_bytes; + kernel_uint_t io_cancelled_write_bytes; int *target_fds; int target_fds_size; - kernel_uint_t openfiles; - kernel_uint_t openpipes; - kernel_uint_t opensockets; - kernel_uint_t openinotifies; - kernel_uint_t openeventfds; - kernel_uint_t opentimerfds; - kernel_uint_t opensignalfds; - kernel_uint_t openeventpolls; - kernel_uint_t openother; + struct openfds openfds; kernel_uint_t starttime; kernel_uint_t collected_starttime; @@ -382,22 +391,24 @@ struct pid_stat { kernel_uint_t io_logical_bytes_read_raw; kernel_uint_t io_logical_bytes_written_raw; - // kernel_uint_t io_read_calls_raw; - // kernel_uint_t io_write_calls_raw; + kernel_uint_t io_read_calls_raw; + kernel_uint_t io_write_calls_raw; kernel_uint_t io_storage_bytes_read_raw; kernel_uint_t io_storage_bytes_written_raw; - // kernel_uint_t io_cancelled_write_bytes_raw; + kernel_uint_t io_cancelled_write_bytes_raw; kernel_uint_t io_logical_bytes_read; kernel_uint_t io_logical_bytes_written; - // kernel_uint_t io_read_calls; - // kernel_uint_t io_write_calls; + kernel_uint_t io_read_calls; + kernel_uint_t io_write_calls; kernel_uint_t io_storage_bytes_read; kernel_uint_t io_storage_bytes_written; - // kernel_uint_t io_cancelled_write_bytes; + kernel_uint_t io_cancelled_write_bytes; struct pid_fd *fds; // array of fds it uses - size_t fds_size; // the size of the fds array + size_t fds_size; // the size of the fds array + + struct openfds openfds; int children_count; // number of processes directly referencing this unsigned char keep:1; // 1 when we need to keep this process in memory even after it exited @@ -447,8 +458,8 @@ kernel_uint_t global_uptime; static struct pid_stat *root_of_pids = NULL, // global list of all processes running - **all_pids = NULL; // to avoid allocations, we pre-allocate the - // the entire pid space. + **all_pids = NULL; // to avoid allocations, we pre-allocate + // a pointer for each pid in the entire pid space. static size_t all_pids_count = 0; // the number of processes running @@ -961,15 +972,10 @@ static inline struct pid_stat *get_pid_entry(pid_t pid) { p->fds = mallocz(sizeof(struct pid_fd) * MAX_SPARE_FDS); p->fds_size = MAX_SPARE_FDS; init_pid_fds(p, 0, p->fds_size); - - if(likely(root_of_pids)) - root_of_pids->prev = p; - - p->next = root_of_pids; - root_of_pids = p; - p->pid = pid; + DOUBLE_LINKED_LIST_APPEND_UNSAFE(root_of_pids, p, prev, next); + all_pids[pid] = p; all_pids_count++; @@ -986,11 +992,7 @@ static inline void del_pid_entry(pid_t pid) { debug_log("process %d %s exited, deleting it.", pid, p->comm); - if(root_of_pids == p) - root_of_pids = p->next; - - if(p->next) p->next->prev = p->prev; - if(p->prev) p->prev->next = p->next; + DOUBLE_LINKED_LIST_REMOVE_UNSAFE(root_of_pids, p, prev, next); // free the filename #ifndef __FreeBSD__ @@ -1554,21 +1556,21 @@ static inline int read_proc_pid_io(struct pid_stat *p, void *ptr) { #else pid_incremental_rate(io, p->io_logical_bytes_read, str2kernel_uint_t(procfile_lineword(ff, 0, 1))); pid_incremental_rate(io, p->io_logical_bytes_written, str2kernel_uint_t(procfile_lineword(ff, 1, 1))); - // pid_incremental_rate(io, p->io_read_calls, str2kernel_uint_t(procfile_lineword(ff, 2, 1))); - // pid_incremental_rate(io, p->io_write_calls, str2kernel_uint_t(procfile_lineword(ff, 3, 1))); + pid_incremental_rate(io, p->io_read_calls, str2kernel_uint_t(procfile_lineword(ff, 2, 1))); + pid_incremental_rate(io, p->io_write_calls, str2kernel_uint_t(procfile_lineword(ff, 3, 1))); pid_incremental_rate(io, p->io_storage_bytes_read, str2kernel_uint_t(procfile_lineword(ff, 4, 1))); pid_incremental_rate(io, p->io_storage_bytes_written, str2kernel_uint_t(procfile_lineword(ff, 5, 1))); - // pid_incremental_rate(io, p->io_cancelled_write_bytes, str2kernel_uint_t(procfile_lineword(ff, 6, 1))); + pid_incremental_rate(io, p->io_cancelled_write_bytes, str2kernel_uint_t(procfile_lineword(ff, 6, 1))); #endif if(unlikely(global_iterations_counter == 1)) { p->io_logical_bytes_read = 0; p->io_logical_bytes_written = 0; - // p->io_read_calls = 0; - // p->io_write_calls = 0; + p->io_read_calls = 0; + p->io_write_calls = 0; p->io_storage_bytes_read = 0; p->io_storage_bytes_written = 0; - // p->io_cancelled_write_bytes = 0; + p->io_cancelled_write_bytes = 0; } return 1; @@ -1577,11 +1579,11 @@ static inline int read_proc_pid_io(struct pid_stat *p, void *ptr) { cleanup: p->io_logical_bytes_read = 0; p->io_logical_bytes_written = 0; - // p->io_read_calls = 0; - // p->io_write_calls = 0; + p->io_read_calls = 0; + p->io_write_calls = 0; p->io_storage_bytes_read = 0; p->io_storage_bytes_written = 0; - // p->io_cancelled_write_bytes = 0; + p->io_cancelled_write_bytes = 0; return 0; #endif } @@ -1888,7 +1890,7 @@ static inline int file_descriptor_find_or_add(const char *name, uint32_t hash) { else if(likely(strncmp(name, "anon_inode:", 11) == 0)) { const char *t = &name[11]; - if(strcmp(t, "inotify") == 0) type = FILETYPE_INOTIFY; + if(strcmp(t, "inotify") == 0) type = FILETYPE_INOTIFY; else if(strcmp(t, "[eventfd]") == 0) type = FILETYPE_EVENTFD; else if(strcmp(t, "[eventpoll]") == 0) type = FILETYPE_EVENTPOLL; else if(strcmp(t, "[timerfd]") == 0) type = FILETYPE_TIMERFD; @@ -1943,7 +1945,6 @@ static inline void cleanup_negative_pid_fds(struct pid_stat *p) { static inline void init_pid_fds(struct pid_stat *p, size_t first, size_t size) { struct pid_fd *pfd = &p->fds[first], *pfdend = &p->fds[first + size]; - size_t i = first; while(pfd < pfdend) { #ifndef __FreeBSD__ @@ -1951,7 +1952,6 @@ static inline void init_pid_fds(struct pid_stat *p, size_t first, size_t size) { #endif clear_pid_fd(pfd); pfd++; - i++; } } @@ -2904,24 +2904,24 @@ static size_t zero_all_targets(struct target *root) { w->io_logical_bytes_read = 0; w->io_logical_bytes_written = 0; - // w->io_read_calls = 0; - // w->io_write_calls = 0; + w->io_read_calls = 0; + w->io_write_calls = 0; w->io_storage_bytes_read = 0; w->io_storage_bytes_written = 0; - // w->io_cancelled_write_bytes = 0; + w->io_cancelled_write_bytes = 0; // zero file counters if(w->target_fds) { memset(w->target_fds, 0, sizeof(int) * w->target_fds_size); - w->openfiles = 0; - w->openpipes = 0; - w->opensockets = 0; - w->openinotifies = 0; - w->openeventfds = 0; - w->opentimerfds = 0; - w->opensignalfds = 0; - w->openeventpolls = 0; - w->openother = 0; + w->openfds.files = 0; + w->openfds.pipes = 0; + w->openfds.sockets = 0; + w->openfds.inotifies = 0; + w->openfds.eventfds = 0; + w->openfds.timerfds = 0; + w->openfds.signalfds = 0; + w->openfds.eventpolls = 0; + w->openfds.other = 0; } w->collected_starttime = 0; @@ -2956,60 +2956,64 @@ static inline void reallocate_target_fds(struct target *w) { } } -static inline void aggregate_fd_on_target(int fd, struct target *w) { - if(unlikely(!w)) - return; - - if(unlikely(w->target_fds[fd])) { - // it is already aggregated - // just increase its usage counter - w->target_fds[fd]++; - return; - } - - // increase its usage counter - // so that we will not add it again - w->target_fds[fd]++; - - switch(all_files[fd].type) { +static void aggregage_fd_type_on_openfds(FD_FILETYPE type, struct openfds *openfds) { + switch(type) { case FILETYPE_FILE: - w->openfiles++; + openfds->files++; break; case FILETYPE_PIPE: - w->openpipes++; + openfds->pipes++; break; case FILETYPE_SOCKET: - w->opensockets++; + openfds->sockets++; break; case FILETYPE_INOTIFY: - w->openinotifies++; + openfds->inotifies++; break; case FILETYPE_EVENTFD: - w->openeventfds++; + openfds->eventfds++; break; case FILETYPE_TIMERFD: - w->opentimerfds++; + openfds->timerfds++; break; case FILETYPE_SIGNALFD: - w->opensignalfds++; + openfds->signalfds++; break; case FILETYPE_EVENTPOLL: - w->openeventpolls++; + openfds->eventpolls++; break; case FILETYPE_OTHER: - w->openother++; + openfds->other++; break; } } +static inline void aggregate_fd_on_target(int fd, struct target *w) { + if(unlikely(!w)) + return; + + if(unlikely(w->target_fds[fd])) { + // it is already aggregated + // just increase its usage counter + w->target_fds[fd]++; + return; + } + + // increase its usage counter + // so that we will not add it again + w->target_fds[fd]++; + + aggregage_fd_type_on_openfds(all_files[fd].type, &w->openfds); +} + static inline void aggregate_pid_fds_on_targets(struct pid_stat *p) { if(unlikely(!p->updated)) { @@ -3023,6 +3027,16 @@ static inline void aggregate_pid_fds_on_targets(struct pid_stat *p) { reallocate_target_fds(u); reallocate_target_fds(g); + p->openfds.files = 0; + p->openfds.pipes = 0; + p->openfds.sockets = 0; + p->openfds.inotifies = 0; + p->openfds.eventfds = 0; + p->openfds.timerfds = 0; + p->openfds.signalfds = 0; + p->openfds.eventpolls = 0; + p->openfds.other = 0; + long currentfds = 0; size_t c, size = p->fds_size; struct pid_fd *fds = p->fds; @@ -3033,6 +3047,7 @@ static inline void aggregate_pid_fds_on_targets(struct pid_stat *p) { continue; currentfds++; + aggregage_fd_type_on_openfds(all_files[fd].type, &p->openfds); aggregate_fd_on_target(fd, w); aggregate_fd_on_target(fd, u); @@ -3079,11 +3094,11 @@ static inline void aggregate_pid_on_target(struct target *w, struct pid_stat *p, w->io_logical_bytes_read += p->io_logical_bytes_read; w->io_logical_bytes_written += p->io_logical_bytes_written; - // w->io_read_calls += p->io_read_calls; - // w->io_write_calls += p->io_write_calls; + w->io_read_calls += p->io_read_calls; + w->io_write_calls += p->io_write_calls; w->io_storage_bytes_read += p->io_storage_bytes_read; w->io_storage_bytes_written += p->io_storage_bytes_written; - // w->io_cancelled_write_bytes += p->io_cancelled_write_bytes; + w->io_cancelled_write_bytes += p->io_cancelled_write_bytes; w->processes++; w->num_threads += p->num_threads; @@ -3638,7 +3653,7 @@ static void send_collected_data_to_netdata(struct target *root, const char *type send_BEGIN(type, "files", dt); for (w = root; w; w = w->next) { if (unlikely(w->exposed && w->processes)) - send_SET(w->name, w->openfiles); + send_SET(w->name, w->openfds.files); } if (!strcmp("apps", type)){ kernel_uint_t usedfdpercentage = (kernel_uint_t) ((currentmaxfds * 100) / sysconf(_SC_OPEN_MAX)); @@ -3649,14 +3664,14 @@ static void send_collected_data_to_netdata(struct target *root, const char *type send_BEGIN(type, "sockets", dt); for (w = root; w; w = w->next) { if (unlikely(w->exposed && w->processes)) - send_SET(w->name, w->opensockets); + send_SET(w->name, w->openfds.sockets); } send_END(); send_BEGIN(type, "pipes", dt); for (w = root; w; w = w->next) { if (unlikely(w->exposed && w->processes)) - send_SET(w->name, w->openpipes); + send_SET(w->name, w->openfds.pipes); } send_END(); } @@ -3704,30 +3719,36 @@ static void send_charts_updates_to_netdata(struct target *root, const char *type if(unlikely(w->exposed)) fprintf(stdout, "DIMENSION %s '' absolute 1 %llu %s\n", w->name, time_factor * RATES_DETAIL / 100, w->hidden ? "hidden" : ""); } + APPS_PLUGIN_FUNCTIONS(); fprintf(stdout, "CHART %s.mem '' '%s Real Memory (w/o shared)' 'MiB' mem %s.mem stacked 20003 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) fprintf(stdout, "DIMENSION %s '' absolute %ld %ld\n", w->name, 1L, 1024L); } + APPS_PLUGIN_FUNCTIONS(); + fprintf(stdout, "CHART %s.vmem '' '%s Virtual Memory Size' 'MiB' mem %s.v |