标签：active option 环境 数据 form orm 默认 paxos his
本章主要介绍mon选举的过程。整个选举过程，在Elector类中实现。此类之中实现了一个election_epoch:
当这个election_epoch为偶数的时候，表示处于稳定状态，为奇数的时候，表示还在选举过程中，mon leader 还未定。
以下是自己环境的mon情况，当前 "election_epoch": 120，说明目前处于稳定状态，没有在选举。
[root@ceph1 ~]# ceph quorum_status | json_reformat
{
"election_epoch": 120,
"quorum": [
0,
1,
2
],
"quorum_names": [
"ceph2",
"ceph1",
"ceph3"
],
"quorum_leader_name": "ceph2",
"monmap": {
"epoch": 1,
"fsid": "5cd6b9b7-2ebd-47b1-a1d2-7362bb077fa3",
"modified": "2020-08-04 19:55:21.613792",
"created": "2020-08-04 19:55:21.613792",
"features": {
"persistent": [
"kraken",
"luminous"
],
"optional": [
]
},
"mons": [
{
"rank": 0,
"name": "ceph2",
"addr": "192.168.111.14:6789/0",
"public_addr": "192.168.111.14:6789/0"
},
{
"rank": 1,
"name": "ceph1",
"addr": "192.168.111.20:6789/0",
"public_addr": "192.168.111.20:6789/0"
},
{
"rank": 2,
"name": "ceph3",
"addr": "192.168.111.34:6789/0",
"public_addr": "192.168.111.34:6789/0"
}
]
}
}
[root@ceph1 ~]#
a.acked_me
/**
* Set containing all those that acked our proposal to become the Leader.
*
* If we are acked by everyone in the MonMap, we will declare
* victory. Also note each peer‘s feature set.
*/
//发起选举本端，经过选举，其他所有mon 返回ack消息，承认选自己为leader
map<int, elector_info_t> acked_me;
b.leader_acked
/**
* Indicates who we have acked
*/
本端已经acked，承认别人是leader
int leader_acked;
/*
默认值-1，代表从未承认别人是leader。
如果defer 了mon.1，那么leader_acked=1.
Elector::defer(int who)函数中进行赋值
*/
c.electing_me
/**
* Indicates if we are the ones being elected.
*
* We always attempt to be the one being elected if we are the ones starting
* the election. If we are not the ones that started it, we will only attempt
* to be elected if we think we might have a chance (i.e., the other guy‘s
* rank is lower than ours).
*/
是否选取本端作为leader,选举发起端一发起，该值就置位true
bool electing_me;
d.epoch
/**
* Latest epoch we‘ve seen.
*
* @remarks if its value is odd, we‘re electing; if it‘s even, then we‘re
* stable.
*/
//选举的版本，奇数为选举状态，偶数为稳定状态
epoch_t epoch;以下开始介绍选举过程：
注意，这里leader端和peon端都可以发起选举。
void Monitor::start_election()
{
dout(10) << "start_election" << dendl;
wait_for_paxos_write();
_reset();
state = STATE_ELECTING;//设置状态
logger->inc(l_mon_num_elections);
logger->inc(l_mon_election_call);
clog->info() << "mon." << name << " calling monitor election";
elector.call_election();
}
void call_election() {
start();
}
void Elector::start()
{
dout(5) << "start -- can i be leader?" << dendl;
acked_me.clear();
init();
// start by trying to elect me /*从稳定态进入选举态，需要将版本号从偶数往上抬，抬成奇数*/
if (epoch % 2 == 0) {
bump_epoch(epoch+1); // odd == election cycle
} else {
// do a trivial db write just to ensure it is writeable.
auto t(std::make_shared<MonitorDBStore::Transaction>());
t->put(Monitor::MONITOR_NAME, "election_writeable_test", rand());
int r = mon->store->apply_transaction(t);
assert(r >= 0);
}
start_stamp = ceph_clock_now();
electing_me = true;
acked_me[mon->rank].cluster_features = CEPH_FEATURES_ALL;
acked_me[mon->rank].mon_features = ceph::features::mon::get_supported();
mon->collect_metadata(&acked_me[mon->rank].metadata);
leader_acked = -1;
// bcast to everyone else /*向每一个成员广播消息，提议开始重新选举*/
for (unsigned i=0; i<mon->monmap->size(); ++i) {
if ((int)i == mon->rank) continue;
MMonElection *m =
new MMonElection(MMonElection::OP_PROPOSE, epoch, mon->monmap);// OP_PROPOSE
m->mon_features = ceph::features::mon::get_supported();
mon->messenger->send_message(m, mon->monmap->get_inst(i)); //发送选举信号
}
/*
*注意，如果所有的人都承认自己leader地位，那么可以宣布获胜。但是有些情况下，无法等到所有的回应。
*比如某个ceph-mon进程已经不在了，是不可能得到其承认的。为了防止出现这种情况下，在通知其他节点
*选自己的start函数设置了定时器“
*/
reset_timer();
} 当选举超时后，会进入expire函数。
void Elector::expire()
{
dout(5) << "election timer expired" << dendl;
// did i win? 从这里可以看到只要有超过一半的monitor 回复，仍然认为当前节点获胜为leader节点，否则就调用mon->bootstrap() 重新开始选举
if (electing_me &&
acked_me.size() > (unsigned)(mon->monmap->size() / 2)) {
// i win
victory();
} else {
// whoever i deferred to didn‘t declare victory quickly enough.
if (mon->has_ever_joined)
start();
else
mon->bootstrap();
}
}消息发送后，会根据不同的消息类型，经过以下流程进行处理。注意这里一次选举过程，可能要经过N次以下处理流程。
bool ms_dispatch(Message *m) ->Monitor::_ms_dispatch(Message *m)-> Monitor::dispatch_op -> Elector::dispatch(MonOpRequestRef op) ->handle_propose| handle_ack|handle_victory|handle_nakvoid Elector::dispatch(MonOpRequestRef op)
{
//选举的地址如果是非monmap中的地址，那么就退出
if (!mon->monmap->contains(em->get_source_addr())) {
dout(1) << "discarding election message: " << em->get_source_addr()
<< " not in my monmap " << *mon->monmap << dendl;
return;
}
...
switch (em->op) {
case MMonElection::OP_PROPOSE://发起选举提议消息
handle_propose(op);
return;
}
if (em->epoch < epoch) {
dout(5) << "old epoch, dropping" << dendl;
break;
}
switch (em->op) {
case MMonElection::OP_ACK://返回ack消息到发起端
handle_ack(op);
return;
case MMonElection::OP_VICTORY://选举成功，leader端发起消息
handle_victory(op);//peon端进行处理
return;
case MMonElection::OP_NAK:
handle_nak(op);
return;
}//处理发起propose的mon端发过来的消息。
void Elector::handle_propose(MonOpRequestRef op)
{
op->mark_event("elector:handle_propose");
MMonElection *m = static_cast<MMonElection*>(op->get_req());
dout(5) << "handle_propose from " << m->get_source() << dendl;
int from = m->get_source().num();
assert(m->epoch % 2 == 1); // election
uint64_t required_features = mon->get_required_features();
mon_feature_t required_mon_features = mon->get_required_mon_features();
dout(10) << __func__ << " required features " << required_features
<< " " << required_mon_features
<< ", peer features " << m->get_connection()->get_features()
<< " " << m->mon_features
<< dendl;
if ((required_features ^ m->get_connection()->get_features()) &
required_features) {
dout(5) << " ignoring propose from mon" << from
<< " without required features" << dendl;
nak_old_peer(op);
return;
} else if (!m->mon_features.contains_all(required_mon_features)) {
// all the features in ‘required_mon_features‘ not in ‘m->mon_features‘
mon_feature_t missing = required_mon_features.diff(m->mon_features);
dout(5) << " ignoring propose from mon." << from
<< " without required mon_features " << missing
<< dendl;
nak_old_peer(op);
} else if (m->epoch > epoch) {
bump_epoch(m->epoch);//正常选举流程
} else if (m->epoch < epoch) {//说明是较老的propose
// got an "old" propose,
if (epoch % 2 == 0 && // in a non-election cycle
mon->quorum.count(from) == 0) { // from someone outside the quorum
// a mon just started up, call a new election so they can rejoin!
dout(5) << " got propose from old epoch, quorum is " << mon->quorum
<< ", " << m->get_source() << " must have just started" << dendl;
// we may be active; make sure we reset things in the monitor appropriately.
mon->start_election();
} else {
dout(5) << " ignoring old propose" << dendl;
return;
}
}
if (mon->rank < from) {//如果从未收到过更强者(rank更小者)发来的选举请求，调用start_election，给所有成员发消息，让他们选自己为mon leader
// i would win over them.
if (leader_acked >= 0) { // we already acked someone 自己曾经认过怂，消息来源的mon还不如自己，直接不理
assert(leader_acked < from); // and they still win, of course
dout(5) << "no, we already acked " << leader_acked << dendl;
} else {
// wait, i should win!
/*注意，electing_me记录了自己是否发出过选我为leader的请求
*如果先后收到两个弱小者发来的选举请求，处理第一个的时候，本节点已经发出了选自己当leader的请求，
*当第二个弱者消息到来的时候，没必要再发送选自己当leader的请求*/
if (!electing_me) {
mon->start_election();
}
}
} else {//赞同其他人做leader
// they would win over me
if (leader_acked < 0 || // haven‘t acked anyone yet, or /*从未承认过别人，从未认过怂*/
leader_acked > from || // they would win over who you did ack, or /*虽然承认过别人，认过怂，无奈这次来的更强大，所以还是得认怂，承认它*/
leader_acked == from) { // this is the guy we‘re already deferring to
defer(from); /*defer函数的作用是认可对方可以当leader*/
} else {//leader_acked < from
// ignore them! /*曾经认可过更强者，不可能向不够强的mon发送认可，不理*/
dout(5) << "no, we already acked " << leader_acked << dendl;
}
}
}
void Elector::bump_epoch(epoch_t e)
{
dout(10) << "bump_epoch " << epoch << " to " << e << dendl;
assert(epoch <= e);
epoch = e;//epoch 奇数
auto t(std::make_shared<MonitorDBStore::Transaction>());
t->put(Monitor::MONITOR_NAME, "election_epoch", epoch);
mon->store->apply_transaction(t);
mon->join_election();
// clear up some state
electing_me = false;//重置选举状态
acked_me.clear();//重置选举状态
}上述handle_propose函数是判断选举条件的核心。已经注释了关键的信息。
大家可能会发现这么个现象，mon发起选举，有时所有mon中都会有选举自己的信息(见下)，有时候部分日志中有，部分日志中没有。这个其实和发起选举的序号大小有关。
mon.ceph2 calling monitor election
if (mon->rank < from) {
// i would win over them.
if (leader_acked >= 0) {//说明认怂过，此时也不可能选举自己
assert(leader_acked < from); // and they still win, of course
dout(5) << "no, we already acked " << leader_acked << dendl;
} else {
// wait, i should win!
if (!electing_me) {
mon->start_election();//本端发起选举
}
}
}此阶段，关键处理函数为defer()，该函数可能被调用多次。如果本端mon defer过序号更小的mon，当再来次小的mon，就不再调用defer() 了。
if (mon->rank < from) {
...
} else {//赞同其他人做leader
// they would win over me
if (leader_acked < 0 || // haven‘t acked anyone yet, or /*从未承认过别人，从未认过怂*/
leader_acked > from || // they would win over who you did ack, or /*虽然承认过别人，认过怂，无奈这次来的更强大，所以还是得认怂，承认它*/
leader_acked == from) { // this is the guy we‘re already deferring to
defer(from); /*defer函数的作用是认可对方可以当leader*/
} else {//leader_acked < from
// ignore them! /*曾经认可过更强者，不可能向不够强的mon发送认可，不理*/
dout(5) << "no, we already acked " << leader_acked << dendl;
}void Elector::defer(int who)
{
dout(5) << "defer to " << who << dendl;
if (electing_me) { /*注意，认怂就要清零，即使自己曾经要求别人选过自己*/
// drop out
acked_me.clear();
electing_me = false;
}
// ack them
leader_acked = who;
ack_stamp = ceph_clock_now();
MMonElection *m = new MMonElection(MMonElection::OP_ACK, epoch, mon->monmap);// OP_ACK /*发送OP_ACK承认对方可以当leader*/
m->mon_features = ceph::features::mon::get_supported();
mon->collect_metadata(&m->metadata);
// This field is unused completely in luminous, but jewel uses it to
// determine whether we are a dumpling mon due to some crufty old
// code. It only needs to see this buffer non-empty, so put
// something useless there.
m->sharing_bl = mon->get_local_commands_bl(mon->get_required_mon_features());
mon->messenger->send_message(m, mon->monmap->get_inst(who));//发送消息
// set a timer
reset_timer(1.0); // give the leader some extra time to declare victory
}基于阶段3，如果非发起端认怂后，会发送OP_ACK消息，那么发起端要进行对应消息的处理。主要处理函数为 Elector::handle_ack()，一般发生在leader端（临时优先级更高的mon端）。
void Elector::handle_ack(MonOpRequestRef op)
{
op->mark_event("elector:handle_ack");
MMonElection *m = static_cast<MMonElection*>(op->get_req());
dout(5) << "handle_ack from " << m->get_source() << dendl;
int from = m->get_source().num();
assert(m->epoch % 2 == 1); // election
assert(m->epoch == epoch);
if (electing_me) {
// thanks
acked_me[from].cluster_features = m->get_connection()->get_features();
acked_me[from].mon_features = m->mon_features;
acked_me[from].metadata = m->metadata;
dout(5) << " so far i have {";
for (map<int, elector_info_t>::const_iterator p = acked_me.begin();
p != acked_me.end();
++p) {
if (p != acked_me.begin())
*_dout << ",";
*_dout << " mon." << p->first << ":"
<< " features " << p->second.cluster_features
<< " " << p->second.mon_features;
}
*_dout << " }" << dendl;
// is that _everyone_? /*如果所有成员都承认了自己的leader地位，那么宣布获胜，调用victory*/
/*
*注意，如果所有的人都承认自己leader地位，那么可以宣布获胜。但是有些情况下，无法等到所有的回应。
*比如某个ceph-mon进程已经不在了，是不可能得到其承认的。为了防止出现这种情况下，在通知其他节点
*选自己的start函数设置了定时器“
*/
if (acked_me.size() == mon->monmap->size()) {//如果所有的mon端都进行了ack，说明都defer了自己，自己成为了leader
// if yes, shortcut to election finish
victory();
}
} else {
// ignore, i‘m deferring already.
assert(leader_acked >= 0);
}
}当发起端接收到所有mon的ack信息后，开始调用victory（）函数，说明本端成为了leader。继续分析
void Elector::victory()
{
leader_acked = -1;//已经选举成功了，那么把相关标记信息复位
electing_me = false;//同上
cancel_timer();
assert(epoch % 2 == 1); // election
bump_epoch(epoch+1); // is over! 这里epoch变为偶数，稳定状态。
// tell everyone!
for (set<int>::iterator p = quorum.begin();
p != quorum.end();
++p) {
if (*p == mon->rank) continue;
MMonElection *m = new MMonElection(MMonElection::OP_VICTORY, epoch,/*注意这里会发送OP_VICTORY 消息来帮自己设置为leader节点*/
mon->monmap);
m->quorum = quorum;
m->quorum_features = cluster_features;
m->mon_features = mon_features;
m->sharing_bl = mon->get_local_commands_bl(mon_features);
mon->messenger->send_message(m, mon->monmap->get_inst(*p));
}
// tell monitor
mon->win_election(epoch, quorum,
cluster_features, mon_features, metadata);
}
void Monitor::win_election(epoch_t epoch, set<int>& active, uint64_t features,
const mon_feature_t& mon_features,
const map<int,Metadata>& metadata)
{
assert(is_electing());
state = STATE_LEADER;// 修改选举状态
clog->info() << "mon." << name << " is new leader, mons " << get_quorum_names()
<< " in quorum (ranks " << quorum << ")";
}
paxos->leader_init();//函数里paoxs状态变为， state=STATE_RECOVERING，开始进行数据恢复操作
monmon()->election_finished();
_finish_svc_election();
health_monitor->start(epoch); //OP_VICTORY消息对应的处理函数
void Elector::handle_victory(MonOpRequestRef op)
{
op->mark_event("elector:handle_victory");
MMonElection *m = static_cast<MMonElection*>(op->get_req());
dout(5) << "handle_victory from " << m->get_source()
<< " quorum_features " << m->quorum_features
<< " " << m->mon_features
<< dendl;
int from = m->get_source().num();
assert(from < mon->rank);// 已经选举成功了，远端的rank自然是小的
assert(m->epoch % 2 == 0); //victory 函数中已经对epoch+1 了
leader_acked = -1;
bump_epoch(m->epoch);//设置本端的epoch以及重置选举状态。此时非leader端mon也进入稳定状态
// they win 输掉选举的话，把自己设置为peon节点
mon->lose_election(epoch, m->quorum, from,
m->quorum_features, m->mon_features);
// cancel my timer
cancel_timer();
// stash leader‘s commands
assert(m->sharing_bl.length());
vector<MonCommand> new_cmds;
bufferlist::iterator bi = m->sharing_bl.begin();
MonCommand::decode_vector(new_cmds, bi);
mon->set_leader_commands(new_cmds);
}
void Monitor::lose_election(epoch_t epoch, set<int> &q, int l,
uint64_t features,
const mon_feature_t& mon_features)
{
dout(10) << "lose_election, epoch " << epoch << " leader is mon" << leader
<< " quorum is " << quorum << " features are " << quorum_con_features
<< " mon_features are " << quorum_mon_features
<< dendl;
state = STATE_PEON;//设置peon状态
paxos->peon_init();
_finish_svc_election();
health_monitor->start(epoch);
}
void Paxos::peon_init()
{
cancel_events();
new_value.clear();
state = STATE_RECOVERING;// peon 端的paxos也进入STATE_RECOVERING状态
}标签：active option 环境 数据 form orm 默认 paxos his
原文地址：https://blog.51cto.com/wendashuai/2519254
