11 #include <sys/resource.h>
21 #define STORM_INTERVAL 2
34 time_t last_election_loss;
48 static gboolean election_timer_cb(gpointer user_data)
52 crm_info(
"%s timed out, declaring local node as winner", e->name);
87 static guint count = 0;
93 crm_perror(LOG_CRIT,
"Cannot create election");
97 e->uname = strdup(
uname);
98 if (e->uname == NULL) {
99 crm_perror(LOG_CRIT,
"Cannot create election");
108 election_timer_cb, e);
125 if(e &&
uname && e->voted) {
126 crm_trace(
"Discarding %s (no-)vote from lost peer %s", e->name,
uname);
127 g_hash_table_remove(e->voted,
uname);
140 crm_trace(
"Resetting election %s", e->name);
143 crm_trace(
"Destroying voted cache with %d members", g_hash_table_size(e->voted));
144 g_hash_table_destroy(e->voted);
204 crm_err(
"No election defined");
209 crm_uptime(
struct timeval *output)
211 static time_t expires = 0;
212 static struct rusage info;
214 time_t tm_now = time(NULL);
216 if (expires < tm_now) {
219 info.ru_utime.tv_sec = 0;
220 info.ru_utime.tv_usec = 0;
221 rc = getrusage(RUSAGE_SELF, &info);
227 crm_perror(LOG_ERR,
"Could not calculate the current uptime");
232 crm_debug(
"Current CPU usage is: %lds, %ldus", (
long)info.ru_utime.tv_sec,
233 (
long)info.ru_utime.tv_usec);
237 output->tv_sec = info.ru_utime.tv_sec;
238 output->tv_usec = info.ru_utime.tv_usec;
244 crm_compare_age(
struct timeval your_age)
246 struct timeval our_age;
248 crm_uptime(&our_age);
250 if (our_age.tv_sec > your_age.tv_sec) {
251 crm_debug(
"Win: %ld vs %ld (seconds)", (
long)our_age.tv_sec, (
long)your_age.tv_sec);
253 }
else if (our_age.tv_sec < your_age.tv_sec) {
254 crm_debug(
"Lose: %ld vs %ld (seconds)", (
long)our_age.tv_sec, (
long)your_age.tv_sec);
256 }
else if (our_age.tv_usec > your_age.tv_usec) {
257 crm_debug(
"Win: %ld.%06ld vs %ld.%06ld (usec)",
258 (
long)our_age.tv_sec, (
long)our_age.tv_usec, (
long)your_age.tv_sec, (
long)your_age.tv_usec);
260 }
else if (our_age.tv_usec < your_age.tv_usec) {
261 crm_debug(
"Lose: %ld.%06ld vs %ld.%06ld (usec)",
262 (
long)our_age.tv_sec, (
long)our_age.tv_usec, (
long)your_age.tv_sec, (
long)your_age.tv_usec);
286 xmlNode *vote = NULL;
290 crm_trace(
"Election vote requested, but no election available");
296 crm_trace(
"Cannot vote in %s yet: local node not connected to cluster",
315 crm_debug(
"Started %s round %d", e->name, e->count);
316 election_timeout_start(e);
342 crm_trace(
"Election check requested, but no election available");
345 if (e->voted == NULL) {
346 crm_trace(
"%s check requested, but no votes received yet", e->name);
350 voted_size = g_hash_table_size(e->voted);
357 if (voted_size >= num_members) {
360 if (voted_size > num_members) {
361 GHashTableIter gIter;
365 crm_warn(
"Received too many votes in %s", e->name);
367 while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) & node)) {
369 crm_warn(
"* expected vote: %s", node->uname);
373 g_hash_table_iter_init(&gIter, e->voted);
374 while (g_hash_table_iter_next(&gIter, (gpointer *) & key, NULL)) {
380 crm_info(
"%s won by local node", e->name);
381 election_complete(e);
385 crm_debug(
"%s still waiting on %d of %d votes",
386 e->name, num_members - voted_size, num_members);
392 #define LOSS_DAMPEN 2
398 const char *election_owner;
415 parse_election_message(
election_t *e, xmlNode *message,
struct vote *vote)
417 CRM_CHECK(message && vote,
return FALSE);
419 vote->election_id = -1;
420 vote->age.tv_sec = -1;
421 vote->age.tv_usec = -1;
430 if ((vote->op == NULL) || (vote->from == NULL) || (vote->version == NULL)
431 || (vote->election_owner == NULL) || (vote->election_id < 0)) {
433 crm_warn(
"Invalid %s message from %s in %s ",
434 (vote->op? vote->op :
"election"),
435 (vote->from? vote->from :
"unspecified node"),
436 (e? e->name :
"election"));
446 if ((vote->age.tv_sec < 0) || (vote->age.tv_usec < 0)) {
447 crm_warn(
"Cannot count %s %s from %s because it is missing uptime",
448 (e? e->name :
"election"), vote->op, vote->from);
453 crm_info(
"Cannot process %s message from %s because %s is not a known election op",
454 (e? e->name :
"election"), vote->from, vote->op);
461 crm_info(
"Cannot count %s from %s because no election available",
462 vote->op, vote->from);
470 crm_info(
"Cannot count %s %s from %s because no peer information available",
471 e->name, vote->op, vote->from);
480 char *voter_copy = NULL;
481 char *vote_copy = NULL;
483 CRM_ASSERT(e && vote && vote->from && vote->op);
484 if (e->voted == NULL) {
485 e->voted = crm_str_table_new();
488 voter_copy = strdup(vote->from);
489 vote_copy = strdup(vote->op);
492 g_hash_table_replace(e->voted, voter_copy, vote_copy);
496 send_no_vote(
crm_node_t *peer,
struct vote *vote)
528 int log_level = LOG_INFO;
529 gboolean done = FALSE;
530 gboolean we_lose = FALSE;
531 const char *reason =
"unknown";
532 bool we_are_owner = FALSE;
533 crm_node_t *our_node = NULL, *your_node = NULL;
534 time_t tm_now = time(NULL);
538 if (parse_election_message(e, message, &vote) == FALSE) {
544 we_are_owner = (our_node != NULL)
547 if(can_win == FALSE) {
548 reason =
"Not eligible";
552 reason =
"We are not part of the cluster";
556 }
else if (we_are_owner && (vote.election_id != e->count)) {
558 reason =
"Superseded";
563 reason =
"Peer is not part of our cluster";
564 log_level = LOG_WARNING;
573 crm_warn(
"Cannot count %s round %d %s from %s because we are not election owner (%s)",
574 e->name, vote.election_id, vote.op, vote.from,
575 vote.election_owner);
580 crm_debug(
"Not counting %s round %d %s from %s because no election in progress",
581 e->name, vote.election_id, vote.op, vote.from);
584 record_vote(e, &vote);
590 int age_result = crm_compare_age(vote.age);
593 if (version_result < 0) {
597 }
else if (version_result > 0) {
600 }
else if (age_result < 0) {
604 }
else if (age_result > 0) {
607 }
else if (strcasecmp(e->uname, vote.from) > 0) {
608 reason =
"Host name";
612 reason =
"Host name";
616 if (e->expires < tm_now) {
617 e->election_wins = 0;
620 }
else if (done == FALSE && we_lose == FALSE) {
627 if (e->election_wins > (peers * peers)) {
628 crm_warn(
"%s election storm detected: %d wins in %d seconds",
630 e->election_wins = 0;
632 if (e->wrote_blackbox == FALSE) {
644 e->wrote_blackbox = TRUE;
651 "Processed %s round %d %s (current round %d) from %s (%s)",
652 e->name, vote.election_id, vote.op, e->count, vote.from,
656 }
else if (we_lose == FALSE) {
670 if ((e->last_election_loss == 0)
671 || ((tm_now - e->last_election_loss) > (time_t)
LOSS_DAMPEN)) {
673 do_crm_log(log_level,
"%s round %d (owner node ID %s) pass: %s from %s (%s)",
674 e->name, vote.election_id, vote.election_owner, vote.op,
677 e->last_election_loss = 0;
684 char *loss_time = ctime(&e->last_election_loss);
691 crm_info(
"Ignoring %s round %d (owner node ID %s) pass vs %s because we lost less than %ds ago at %s",
692 e->name, vote.election_id, vote.election_owner, vote.from,
697 e->last_election_loss = tm_now;
699 do_crm_log(log_level,
"%s round %d (owner node ID %s) lost: %s from %s (%s)",
700 e->name, vote.election_id, vote.election_owner, vote.op,
704 send_no_vote(your_node, &vote);
717 e->last_election_loss = 0;