Explanation of lab2a experiment
1. raft structure complementary field of raft.go.Fields should be as close as possible to Figure2 from raft's paper.
type Raft struct {
mu sync.Mutex // Lock to protect shared access to this peer's state
peers []*labrpc.ClientEnd // RPC end points of all peers
persister *Persister // Object to hold this peer's persisted state
me int // this peer's index into peers[]
dead int32 // set by Kill()
// Your data here (2A, 2B, 2C).
// Look at the paper's Figure 2 for a description of what
// state a Raft server must maintain.
state int // follower, candidate or leader
resetTimer chan struct{} // for reset election timer
electionTimer *time.Timer // election timer
electionTimeout time.Duration // 400~800ms
heartbeatInterval time.Duration // 100ms
CurrentTerm int // Persisted before responding to RPCs
VotedFor int // Persisted before responding to RPCs
Logs []LogEntry // Persisted before responding to RPCs
commitCond *sync.Cond // for commitIndex update
//newEntryCond []*sync.Cond // for new log entry
commitIndex int // Volatile state on all servers
lastApplied int // Volatile state on all servers
nextIndex []int // Leader only, reinitialized after election
matchIndex []int // Leader only, reinitialized after election
applyCh chan ApplyMsg // outgoing channel to service
shutdownCh chan struct{} // shutdown channel, shut raft instance gracefully
}Get the term and state of the current raft node
func (rf *Raft) GetState() (int, bool) {
var term int
var isleader bool
// Your code here (2A).
rf.mu.Lock()
defer rf.mu.Unlock()
term = rf.CurrentTerm
isleader = rf.state == Leader
return term, isleader
}2. Fill in the RequestVoteArgs and RequestVoteReply structures.
type RequestVoteArgs struct {
// Your data here (2A, 2B).
Term int // candidate's term
CandidateID int // candidate requesting vote
LastLogIndex int // index of candidate's last log entry
LastLogTerm int // term of candidate's last log entry
}
type RequestVoteReply struct {
// Your data here (2A).
CurrentTerm int // currentTerm, for candidate to update itself
VoteGranted bool // true means candidate received vote
}Implement RPC Method RequestVote
1. Get the number of logs of the current node and the term of the last log to determine the term of the current node.
2. If the term of the calling node is less than the current node, return the current term and do not vote for it.
3. If the term of the calling node is larger than the current node, modify the term of the current node and the current node becomes follower.
4. If the term of the calling node is greater than or equal to the current node term and the number of logs of the calling node is greater than or equal to the log of the current node, the calling node votes.
5. The election timeout for resetting the current node after voting.
func (rf *Raft) RequestVote(args *RequestVoteArgs, reply *RequestVoteReply) {
// Your code here (2A, 2B).
select {
case <-rf.shutdownCh:
DPrintf("[%d-%s]: peer %d is shutting down, reject RV rpc request.\n", rf.me, rf, rf.me)
return
default:
}
rf.mu.Lock()
defer rf.mu.Unlock()
lastLogIdx, lastLogTerm := rf.lastLogIndexAndTerm()
DPrintf("[%d-%s]: rpc RV, from peer: %d, arg term: %d, my term: %d (last log idx: %d->%d, term: %d->%d)\n", rf.me, rf, args.CandidateID, args.Term, rf.CurrentTerm, args.LastLogIndex,
lastLogIdx, args.LastLogTerm, lastLogTerm)
if args.Term < rf.CurrentTerm {
reply.CurrentTerm = rf.CurrentTerm
reply.VoteGranted = false
} else {
if args.Term > rf.CurrentTerm {
// convert to follower
rf.CurrentTerm = args.Term
rf.state = Follower
rf.VotedFor = -1
}
// if is null (follower) or itself is a candidate (or stale leader) with same term
if rf.VotedFor == -1 { //|| (rf.VotedFor == rf.me && !sameTerm) { //|| rf.votedFor == args.CandidateID {
// check whether candidate's log is at-least-as update
if (args.LastLogTerm == lastLogTerm && args.LastLogIndex >= lastLogIdx) ||
args.LastLogTerm > lastLogTerm {
rf.resetTimer <- struct{}{}
rf.state = Follower
rf.VotedFor = args.CandidateID
reply.VoteGranted = true
DPrintf("[%d-%s]: peer %d vote to peer %d (last log idx: %d->%d, term: %d->%d)\n",
rf.me, rf, rf.me, args.CandidateID, args.LastLogIndex, lastLogIdx, args.LastLogTerm, lastLogTerm)
}
}
}
}Modify make
In addition to some basic initialization, a new goroutine has been opened.
func Make(peers []*labrpc.ClientEnd, me int,
persister *Persister, applyCh chan ApplyMsg) *Raft {
rf := &Raft{}
rf.peers = peers
rf.persister = persister
rf.me = me
rf.applyCh = applyCh
// Your initialization code here (2A, 2B, 2C).
rf.state = Follower
rf.VotedFor = -1
rf.Logs = make([]LogEntry, 1) // first index is 1
rf.Logs[0] = LogEntry{ // placeholder
Term: 0,
Command: nil,
}
rf.nextIndex = make([]int, len(peers))
rf.matchIndex = make([]int, len(peers))
rf.electionTimeout = time.Millisecond * time.Duration(400+rand.Intn(100)*4)
rf.electionTimer = time.NewTimer(rf.electionTimeout)
rf.resetTimer = make(chan struct{})
rf.shutdownCh = make(chan struct{}) // shutdown raft gracefully
rf.commitCond = sync.NewCond(&rf.mu) // commitCh, a distinct goroutine
rf.heartbeatInterval = time.Millisecond * 40 // small enough, not too small
// initialize from state persisted before a crash
rf.readPersist(persister.ReadRaftState())
go rf.electionDaemon() // kick off election
return rf
}Election Core Selection Daemon
In addition to shutdown, there are two channels, one is the electionTimer, which is used for election timeouts.
One is resetTimer, which resets the election timeout.
Note that time.reset is difficult to use correctly.
Once the election timed out, call go rf.canvassVotes()
// electionDaemon
func (rf *Raft) electionDaemon() {
for {
select {
case <-rf.shutdownCh:
DPrintf("[%d-%s]: peer %d is shutting down electionDaemon.\n", rf.me, rf, rf.me)
return
case <-rf.resetTimer:
if !rf.electionTimer.Stop() {
<-rf.electionTimer.C
}
rf.electionTimer.Reset(rf.electionTimeout)
case <-rf.electionTimer.C:
rf.mu.Lock()
DPrintf("[%d-%s]: peer %d election timeout, issue election @ term %d\n", rf.me, rf, rf.me, rf.CurrentTerm)
rf.mu.Unlock()
go rf.canvassVotes()
rf.electionTimer.Reset(rf.electionTimeout)
}
}
}Voting
The replyHandler is the processing after the request has been returned.
In order for the current node to be a leader, the RequestVote method for each node is invoked.
If the returned term is larger than the current term, the current node becomes a follower, resetting the election timeout.
Otherwise, if more than half of the nodes are voted on, they become leader s and immediately send heartbeat detection to other nodes.
// canvassVotes issues RequestVote RPC
func (rf *Raft) canvassVotes() {
var voteArgs RequestVoteArgs
rf.fillRequestVoteArgs(&voteArgs)
peers := len(rf.peers)
var votes = 1
replyHandler := func(reply *RequestVoteReply) {
rf.mu.Lock()
defer rf.mu.Unlock()
if rf.state == Candidate {
if reply.CurrentTerm > voteArgs.Term {
rf.CurrentTerm = reply.CurrentTerm
rf.turnToFollow()
//rf.persist()
rf.resetTimer <- struct{}{} // reset timer
return
}
if reply.VoteGranted {
if votes == peers/2 {
rf.state = Leader
rf.resetOnElection() // reset leader state
go rf.heartbeatDaemon() // new leader, start heartbeat daemon
DPrintf("[%d-%s]: peer %d become new leader.\n", rf.me, rf, rf.me)
return
}
votes++
}
}
}
for i := 0; i < peers; i++ {
if i != rf.me {
go func(n int) {
var reply RequestVoteReply
if rf.sendRequestVote(n, &voteArgs, &reply) {
replyHandler(&reply)
}
}(i)
}
}
}runtastic Heart Rate PRO
1. leader calls the AppendEntries method for each node.
2. AppendEntries fails if the current node is larger than the calling node.Otherwise, modifying the current term is the largest.
3. If the current node is leader, always make it follower (for leader stability)
4. Vote the current node to the caller (for backward nodes).
5. Time-out to reset the current node.
func (rf *Raft) heartbeatDaemon() {
for {
if _, isLeader := rf.GetState(); !isLeader {
return
}
// reset leader's election timer
rf.resetTimer <- struct{}{}
select {
case <-rf.shutdownCh:
return
default:
for i := 0; i < len(rf.peers); i++ {
if i != rf.me {
go rf.consistencyCheck(i) // routine heartbeat
}
}
}
time.Sleep(rf.heartbeatInterval)
}
}
func (rf *Raft) consistencyCheck(n int) {
rf.mu.Lock()
defer rf.mu.Unlock()
pre := rf.nextIndex[n] - 1
var args = AppendEntriesArgs{
Term: rf.CurrentTerm,
LeaderID: rf.me,
PrevLogIndex: pre,
PrevLogTerm: rf.Logs[pre].Term,
Entries: nil,
LeaderCommit: rf.commitIndex,
}
go func() {
DPrintf("[%d-%s]: consistency Check to peer %d.\n", rf.me, rf, n)
var reply AppendEntriesReply
if rf.sendAppendEntries(n, &args, &reply) {
rf.consistencyCheckReplyHandler(n, &reply)
}
}()
}
func (rf *Raft) AppendEntries(args *AppendEntriesArgs, reply *AppendEntriesReply) {
select {
case <-rf.shutdownCh:
DPrintf("[%d-%s]: peer %d is shutting down, reject AE rpc request.\n", rf.me, rf, rf.me)
return
default:
}
DPrintf("[%d-%s]: rpc AE, from peer: %d, term: %d\n", rf.me, rf, args.LeaderID, args.Term)
rf.mu.Lock()
defer rf.mu.Unlock()
if args.Term < rf.CurrentTerm {
//DPrintf("[%d-%s]: AE failed from leader %d. (heartbeat: leader's term < follower's term (%d < %d))\n",
// rf.me, rf, args.LeaderID, args.Term, rf.currentTerm)
reply.CurrentTerm = rf.CurrentTerm
reply.Success = false
return
}
if rf.CurrentTerm < args.Term {
rf.CurrentTerm = args.Term
}
// for stale leader
if rf.state == Leader {
rf.turnToFollow()
}
// for straggler (follower)
if rf.VotedFor != args.LeaderID {
rf.VotedFor = args.LeaderID
}
// valid AE, reset election timer
// if the node recieve heartbeat. then it will reset the election timeout
rf.resetTimer <- struct{}{}
reply.Success = true
reply.CurrentTerm = rf.CurrentTerm
return
}Handle heartbeat detection return
If the heartbeat detection fails, it becomes a follower and the reset election timed out.
// n: which follower
func (rf *Raft) consistencyCheckReplyHandler(n int, reply *AppendEntriesReply) {
rf.mu.Lock()
defer rf.mu.Unlock()
if rf.state != Leader {
return
}
if reply.Success {
} else {
// found a new leader? turn to follower
if rf.state == Leader && reply.CurrentTerm > rf.CurrentTerm {
rf.turnToFollow()
rf.resetTimer <- struct{}{}
DPrintf("[%d-%s]: leader %d found new term (heartbeat resp from peer %d), turn to follower.",
rf.me, rf, rf.me, n)
return
}
}
}