epoll Timer Implementation Series: Time Wheel for High Performance Timer

Posted by 696020 on Wed, 15 May 2019 22:39:28 +0200

http://blog.csdn.net/jasonliuvip/article/details/23888851

As we mentioned earlier, there is a problem with sorted list-based timers: adding timers is inefficient.The time frame we discuss below solves this problem.

In the figure, this is a simple time wheel:


The solid pointer in the wheel points to a slot on the wheel, which rotates clockwise at a constant speed. Each step of rotation points to the next slot, and each turn is called a tick.

The time of a tick is called slot interval, which is actually the heart beat time.

The wheel has N slots, so it runs for one week at N x si.Each slot points to a timer list, and the timers on each list have the same characteristics: their timing times differ by an integer multiple of N*si.The time wheel uses this relationship to hash timers into different chains.

If the pointer now points to the slot CS and we want to add a timer with a time of ti, the timer will be inserted into the chain table corresponding to ts (timer slot): ts = (cs + (ti / si))%N

A timer based on a sorted list uses a unique list of chains to manage all timers, so the efficiency of insertion decreases as the number of timers increases.The time wheel uses the idea of a hash table to hash timers across different chains.

In this way, the number of timers on each chain table will be significantly reduced, and the efficiency of insertion operation will be less affected by the number of timers.

Obviously, for the time wheel, to improve the timing accuracy, the si value must be small enough; to improve the efficiency of execution, the N value must be large enough.


Let's implement a simple time wheel, just one wheel.Complex time wheels can have multiple, different wheels have different granularities.The two adjacent wheels make a high-precision turn, and the low-precision ones only move forward one slot, just like a water meter.

For a time wheel, the time complexity to add a timer is O(1), to delete a timer is O(1), and to execute a timer is O(n).

However, it is actually much more efficient to execute a timer than O(n), because the time wheel spreads all timers across different chains, the more slots the time wheel has, the more entries the Hash list has, and the fewer timers on each chain.Furthermore, our code only uses one time wheel, and when implemented with multiple wheels, its time complexity will approach O(1).

For the comparison of timer implementation under linux, you can see this article, which is very good: http://www.ibm.com/developerworks/cn/linux/l-cn-timers/


1. Code:

  1. //tw_timer.h  
  2. #ifndef __TIME_WHEEL__  
  3. #define __TIME_WHEEL__  
  4.   
  5. #include <time.h>  
  6. #include <netinet/in.h>  
  7. #include <stdio.h>  
  8.   
  9. #define BUFFER_SIZE 64  
  10.   
  11. class tw_timer;  
  12.   
  13. //Client Data  
  14. struct client_data  
  15. {  
  16.     sockaddr_in address;  
  17.     int sockfd;  
  18.     char buf[BUFFER_SIZE];  
  19.     tw_timer *timer;  
  20. };  
  21.   
  22. //timer  
  23. class tw_timer  
  24. {  
  25. public:  
  26.     tw_timer(int rot, int ts)  
  27.         :next(NULL), prev(NULL), rotation(rot), time_slot(ts) {}  
  28.   
  29. public:  
  30.     int rotation;                       //The timer takes effect after how many turns on the timewheel  
  31.     int time_slot;                      //Which slot on the time wheel does the timer belong to  
  32.   
  33.     void (*cb_func)(client_data*);      //Callback function of timer  
  34.     client_data *user_data;             //Client Data  
  35.       
  36.     tw_timer *prev;                     //Point to the last timer  
  37.     tw_timer *next;                     //Point to Next Timer  
  38. };  
  39.   
  40. //Time Wheel  
  41. class time_wheel  
  42. {  
  43. public:  
  44.     time_wheel();  
  45.     ~time_wheel();  
  46.   
  47.     tw_timer* add_timer(int timeout);   //Create a timer based on the timer value and insert it in the right place  
  48.     void del_timer(tw_timer *timer);    //Delete Target Timer  
  49.     void tick();                //Call this function after time has elapsed, and the time wheel rolls forward one slot interval  
  50.       
  51. private:  
  52.     static const int N = 60;    //Number of slots on the time wheel  
  53.     static const int TI = 1;    //Slot interval time, i.e. one turn per second  
  54.   
  55.     int cur_slot;               //Current groove of time wheel  
  56.     tw_timer *slots[N];         //Slots in the timewheel, where each element points to a timer chain  
  57.   
  58. };  
  59.   
  60.   
  61. #endif  


  1. //tw_timer.cpp  
  2.   
  3. #include "tw_timer.h"  
  4.   
  5. time_wheel::time_wheel():cur_slot(0)  
  6. {  
  7.     //Initialize the head nodes of each slot  
  8.     for (int i = 0; i < N; ++i)  
  9.         slots[i] = NULL;  
  10. }  
  11.   
  12. time_wheel::~time_wheel()  
  13. {  
  14.     //Traverse through each slot and destroy the timer  
  15.     for (int i = 0; i < N; ++i) {  
  16.         tw_timer *tmp = slots[i];  
  17.         while (tmp) {  
  18.             slots[i] = tmp->next;  
  19.             delete tmp;  
  20.             tmp = slots[i];  
  21.         }  
  22.     }  
  23. }  
  24.   
  25. tw_timer* time_wheel::add_timer(int timeout)  
  26. {  
  27.     if (timeout < 0)  
  28.         return NULL;  
  29.   
  30.     int ticks = 0;              //Total ticks required to insert timer  
  31.     if (timeout < TI)  
  32.         ticks = 1;  
  33.     else  
  34.         ticks = timeout / TI;  
  35.   
  36.     int rotation = ticks / N;   //Calculate how many turns the timer to insert will turn on the timewheel before triggering  
  37.     int ts = (cur_slot + ticks) % N; //Calculate where the timer to hold should be inserted  
  38.     //int ts = (cur_slot + (ticks %N)) % N;  
  39.   
  40.     //Create a timer that is triggered after the rotation circle is rotated by the time wheel and is located in slot ts  
  41.     tw_timer *timer = new tw_timer(rotation, ts);  
  42.   
  43.     //If the slot is empty, it has a new timer inserted and set to the head node of the slot  
  44.     if (!slots[ts]) {  
  45.         printf("add timer, rotation is %d, ts is %d, cur_slot is %d\n",  
  46.                 rotation, ts, cur_slot);  
  47.         slots[ts] = timer;  
  48.     }  
  49.     else {  
  50.         timer->next = slots[ts];  
  51.         slots[ts]->prev = timer;  
  52.         slots[ts] = timer;  
  53.     }  
  54.       
  55.     return timer;  
  56. }  
  57.   
  58. void time_wheel::del_timer(tw_timer *timer)  
  59. {  
  60.     if (!timer)  
  61.         return;  
  62.   
  63.     int ts = timer->time_slot;  
  64.     if (timer == slots[ts]) {   //If it is a header node  
  65.         slots[ts] = slots[ts]->next;  
  66.         if (slots[ts])  
  67.             slots[ts]->prev = NULL;  
  68.   
  69.         delete timer;  
  70.     }  
  71.     else {  
  72.         timer->prev->next = timer->next;  
  73.         if (timer->next)  
  74.             timer->next->prev = timer->prev;  
  75.   
  76.         delete timer;  
  77.     }  
  78. }  
  79.   
  80. void time_wheel::tick()  
  81. {  
  82.    //Get the head node of the current slot on the timewheel  
  83.    tw_timer *tmp = slots[cur_slot];  
  84.    printf("current slot is %d\n", cur_slot);  
  85.   
  86.    while (tmp) {  
  87.        printf("tick the timer once\n");  
  88.   
  89.        //If the rotation value of the timer is greater than 0, it is not processed  
  90.        if (tmp->rotation > 0) {  
  91.            tmp->rotation--;  
  92.            tmp = tmp->next;  
  93.        }  
  94.        else {  
  95.            tmp->cb_func(tmp->user_data);  
  96.            if (tmp == slots[cur_slot]) {  
  97.                printf("delete header in cur_slot\n");  
  98.                slots[cur_slot] = tmp->next;  
  99.   
  100.                delete tmp;  
  101.   
  102.                if (slots[cur_slot])  
  103.                    slots[cur_slot]->prev = NULL;  
  104.   
  105.                tmp = slots[cur_slot];  
  106.            }  
  107.            else {  
  108.                tmp->prev->next = tmp->next;  
  109.                if (tmp->next)  
  110.                    tmp->next->prev = tmp->prev;  
  111.   
  112.                tw_timer *tmp2 = tmp->next;  
  113.   
  114.                delete tmp;  
  115.                tmp = tmp2;   
  116.            }  
  117.        }  
  118.    }  
  119.   
  120.    //Update the current slot of the time wheel to reflect the rotation of the time wheel  
  121.    cur_slot = ++cur_slot % N;  
  122. }  
  123.   
  124.       


  1. //nonactive_conn.cpp  
  2. //Close inactive connections  
  3. #include <sys/types.h>  
  4. #include <sys/socket.h>  
  5. #include <netinet/in.h>  
  6. #include <arpa/inet.h>  
  7. #include <stdio.h>  
  8. #include <stdlib.h>  
  9. #include <unistd.h>  
  10. #include <string.h>  
  11. #include <errno.h>  
  12. #include <fcntl.h>  
  13. #include <sys/epoll.h>  
  14. #include <assert.h>  
  15. #include <signal.h>  
  16. #include <pthread.h>  
  17.   
  18. #include "tw_timer.h"  
  19.   
  20. #define FD_LIMIT 65535  
  21. #define MAX_EVENT_NUMBER 1024  
  22. #define TIMESLOT 100  
  23.   
  24. static int pipefd[2];  
  25. static time_wheel client_conn_time_wheel;  
  26. static int epollfd = 0;  
  27.   
  28. int setnonblocking(int fd);             //Set up non-blocking  
  29. int addfd(int epollfd, int fd);         //Add Descriptor Event  
  30.   
  31. void sig_handler(int sig);              //Signal processing function  
  32. void addsig(int sig);                   //Add Signal Processing Function  
  33.   
  34. void timer_handler();                   //TimerTask  
  35. void cb_func(client_data *user_data);   //Timer callback function  
  36.   
  37.   
  38. int main(int argc, char *argv[])  
  39. {  
  40.     if (argc != 2) {  
  41.         fprintf(stderr, "Usage: %s port\n", argv[0]);  
  42.         return 1;  
  43.     }  
  44.   
  45.     int port = atoi(argv[1]);  
  46.   
  47.     int ret = 0;  
  48.     int error;  
  49.   
  50.     struct sockaddr_in address;  
  51.     bzero(&address, sizeof(address));  
  52.     address.sin_family = AF_INET;  
  53.     address.sin_port = htons(port);  
  54.     address.sin_addr.s_addr = htonl(INADDR_ANY);  
  55.   
  56.     int sockfd = socket(PF_INET, SOCK_STREAM, 0);  
  57.     if (sockfd == -1) {  
  58.         fprintf(stderr, "create socket failed\n");  
  59.         return 1;  
  60.     }  
  61.   
  62.     int reuse = 1;  
  63.     ret = setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));  
  64.     if (ret == -1) {  
  65.         error = errno;  
  66.         while ((close(sockfd) == -1) && (errno == EINTR));  
  67.         errno = error;  
  68.         return 1;  
  69.     }  
  70.   
  71.     if ( (bind(sockfd, (struct sockaddr*)&address, sizeof(address)) == -1) ||   
  72.         (listen(sockfd, 5) == -1)) {  
  73.         error = errno;  
  74.         while ((close(sockfd) == -1) && (errno ==  EINTR));  
  75.         errno = error;  
  76.         return 1;  
  77.     }  
  78.   
  79.     int epollfd = epoll_create(5);  
  80.     if (epollfd == -1) {  
  81.         error = errno;  
  82.         while ((close(sockfd)) && (errno == EINTR));  
  83.         errno = error;  
  84.         return 1;  
  85.     }  
  86.   
  87.     ret = socketpair(PF_UNIX, SOCK_STREAM, 0, pipefd);  
  88.     if (ret == -1) {  
  89.         error = errno;  
  90.         while ((close(sockfd) == -1) && (errno == EINTR));  
  91.         errno = error;  
  92.         return 1;  
  93.     }  
  94.   
  95.     epoll_event events[MAX_EVENT_NUMBER];  
  96.       
  97.     setnonblocking(pipefd[1]);  
  98.     addfd(epollfd, pipefd[0]);  
  99.     addfd(epollfd, sockfd);  
  100.   
  101.     //Add Signal Processing  
  102.     addsig(SIGALRM);  
  103.     addsig(SIGTERM);  
  104.   
  105.     bool stop_server = false;  
  106.   
  107.     client_data *users = new client_data[FD_LIMIT];  
  108.     bool timeout = false;  
  109.   
  110.     alarm(1);  
  111.   
  112.     printf("server start...\n");  
  113.   
  114.     while (!stop_server) {  
  115.         int number = epoll_wait(epollfd, events, MAX_EVENT_NUMBER, -1);  
  116.         if (number < 0 && errno != EINTR) {  
  117.             fprintf(stderr, "epoll_wait failed\n");  
  118.             break;  
  119.         }  
  120.   
  121.         for (int i = 0; i < number; i++) {  
  122.             int listenfd = events[i].data.fd;  
  123.               
  124.             if (listenfd == sockfd) {  
  125.                 struct sockaddr_in client_address;  
  126.                 socklen_t client_addrlength = sizeof(client_address);  
  127.   
  128.                 int connfd;  
  129.   
  130.                 while ( ((connfd = accept(listenfd, (struct sockaddr*)&client_address, &client_addrlength)) == -1) &&  
  131.                         (errno == EINTR));  
  132.           
  133.                 addfd(epollfd, connfd);  
  134.   
  135.                 users[connfd].address = client_address;  
  136.                 users[connfd].sockfd = connfd;  
  137.                   
  138.                 tw_timer *timer = NULL;  
  139.                 timer = client_conn_time_wheel.add_timer(TIMESLOT);  
  140.                 if (timer) {  
  141.                     timer->user_data = &users[connfd];  
  142.                     timer->cb_func = cb_func;  
  143.                     users[connfd].timer = timer;  
  144.               fprintf(stderr, "client: %d add tw_timer successed\n", connfd);  
  145.                 }  
  146.                 else {  
  147.                     fprintf(stderr, "client: %d add tw_timer failed\n", connfd);  
  148.                 }  
  149.   
  150.             }  
  151.             else if((listenfd == pipefd[0]) && (events[i].events & EPOLLIN)) {  
  152.                 int sig;  
  153.                 char signals[1024];  
  154.   
  155.                 ret = recv(pipefd[0], signals, sizeof(signals), 0);  
  156.                 if (ret == -1) {  
  157.                     continue;  
  158.                 }  
  159.                 else if (ret == 0) {  
  160.                     continue;  
  161.                 }  
  162.                 else {  
  163.                     for (int i = 0; i < ret; i++) {  
  164.                         switch (signals[i]) {  
  165.                         case SIGALRM:  
  166.                             {  
  167.                                 timeout = true;  
  168.                                 break;  
  169.                             }  
  170.                         case SIGTERM:  
  171.                             {  
  172.                                 stop_server = true;  
  173.                                 break;  
  174.                             }  
  175.                         default:  
  176.                             break;  
  177.                         }  
  178.                     }  
  179.                 }  
  180.             }  
  181.             else if (events[i].events & EPOLLIN) {  
  182.                 memset(users[listenfd].buf, '\0', BUFFER_SIZE);  
  183.   
  184.                 ret = recv(listenfd, users[listenfd].buf, BUFFER_SIZE-1, 0);  
  185.                 printf("get %d bytes of client data: %s from %d\n",  
  186.                         ret, users[listenfd].buf, listenfd);  
  187.   
  188.                 tw_timer *timer = users[listenfd].timer;  
  189.   
  190.                 if (ret < 0) {  
  191.                     if (errno != EAGAIN) {  
  192.                         cb_func(&users[listenfd]);  
  193.                         if (timer)  
  194.                             client_conn_time_wheel.del_timer(timer);  
  195.                     }  
  196.                 }  
  197.                 else if (ret == 0) {  
  198.                     cb_func(&users[listenfd]);  
  199.                     if (timer)  
  200.                         client_conn_time_wheel.del_timer(timer);  
  201.                 }  
  202.                 else {  
  203.                     if (timer) {  
  204.                         printf("conntioned..to do adjuest timer\n");  
  205.                     }  
  206.                 }  
  207.             }  
  208.             else {  
  209.               
  210.             }  
  211.                
  212.         }  
  213.           
  214.         if (timeout) {  
  215.             timer_handler();  
  216.             timeout = false;  
  217.         }  
  218.   
  219.     }  
  220.       
  221.     close(sockfd);  
  222.     close(pipefd[1]);  
  223.     close(pipefd[0]);  
  224.     delete[] users;  
  225.   
  226.   
  227.     return 0;  
  228. }  
  229.   
  230.   
  231. int setnonblocking(int fd)  
  232. {  
  233.     int old_option = fcntl(fd, F_GETFL);  
  234.     int new_option = old_option | O_NONBLOCK;  
  235.     fcntl(fd, F_SETFL, new_option);  
  236.     return old_option;  
  237. }  
  238.   
  239. int addfd(int epollfd, int fd)  
  240. {  
  241.     epoll_event event;  
  242.     event.data.fd = fd;  
  243.     event.events = EPOLLIN | EPOLLET;  
  244.     epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &event);  
  245.     setnonblocking(fd);  
  246. }  
  247.   
  248. void sig_handler(int sig)  
  249. {  
  250.     int save_error = errno;  
  251.     int msg = sig;  
  252.     send(pipefd[1], (char*)&msg, 1, 0);  
  253.     errno = save_error;  
  254. }  
  255.   
  256. void addsig(int sig)  
  257. {  
  258.     struct sigaction sa;  
  259.     memset(&sa, '\0'sizeof(sa));  
  260.     sa.sa_handler = sig_handler;  
  261.     sa.sa_flags |= SA_RESTART;  
  262.   
  263.     sigfillset(&sa.sa_mask);  
  264.       
  265.     assert(sigaction(sig, &sa, NULL) != -1);  
  266. }  
  267.   
  268. void timer_handler()  
  269. {  
  270.     client_conn_time_wheel.tick();  
  271.     alarm(1);  
  272. }  
  273.   
  274. void cb_func(client_data *user_data)  
  275. {  
  276.     epoll_ctl(epollfd, EPOLL_CTL_DEL, user_data->sockfd, 0);  
  277.     assert(user_data);  
  278.     close(user_data->sockfd);  
  279.     printf("close fd %d\n", user_data->sockfd);  
  280. }  


Reference: linux High Performance Server Programming


Topics: socket Linux less Programming