1, Introduction
There are many basic framework software of openwrt routing operating system, most of which are general software modules, such as dhcp, dnsmasq, iproute, cmwp, vpn, ipsec, etc; Openwrt also integrates some software modules with exclusive features, which are also the core framework software components of openwrt system. Starting from this article, we will analyze the series of articles on basic software modules of openwrt system framework.
OpenWrt core software: procd, uci, libubox, ubus, ubox, luci, netifd software component content. This part of the software module is the basic software constituting the OpenWrt framework.
There are many contents involved in the source code of procd. The author's previous articles "a detailed explanation of libubox of the basic software module of OpenWRT system framework" are all prepared to analyze the contents of procd and netifd. Please review the previous contents and read this article.
2, procd software architecture
Let's first look at the source code structure in the procd folder, cmakelist Txt file is the basis for project production. The key contents are as follows:
cmake_minimum_required(VERSION 2.6)
PROJECT(procd C)
INCLUDE(GNUInstallDirs)
ADD_DEFINITIONS(-Os -ggdb -Wall -Werror --std=gnu99 -Wmissing-declarations)
SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "")
IF(APPLE)
INCLUDE_DIRECTORIES(/opt/local/include)
LINK_DIRECTORIES(/opt/local/lib)
ENDIF()
ADD_LIBRARY(setlbf SHARED service/setlbf.c)
INSTALL(TARGETS setlbf
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
SET(SOURCES procd.c signal.c state.c inittab.c rcS.c ubus.c system.c sysupgrade.c
service/service.c service/instance.c service/validate.c service/trigger.c service/watch.c
utils/utils.c)
IF(NOT DISABLE_INIT)
SET(SOURCES ${SOURCES} watchdog.c plug/coldplug.c plug/hotplug.c)
ENDIF()
IF(ZRAM_TMPFS)
ADD_DEFINITIONS(-DZRAM_TMPFS)
SET(SOURCES_ZRAM initd/zram.c)
ENDIF()
add_subdirectory(upgraded)
// First system initialization / sbin/init thread
// The kernel retrieves the evn arg environment variable and executes the user state initialization thread / sbin/init;
IF(DISABLE_INIT)
ADD_DEFINITIONS(-DDISABLE_INIT)
ELSE()
ADD_EXECUTABLE(init initd/init.c initd/early.c initd/preinit.c initd/mkdev.c sysupgrade.c watchdog.c
utils/utils.c ${SOURCES_ZRAM})
TARGET_LINK_LIBRARIES(init ${LIBS})
INSTALL(TARGETS init
RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
)
// Second procd thread
ADD_EXECUTABLE(procd ${SOURCES})
TARGET_LINK_LIBRARIES(procd ${LIBS})
INSTALL(TARGETS procd
RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
)
// Third udevtrigger device daemon thread (udev)
ADD_EXECUTABLE(udevtrigger plug/udevtrigger.c)
INSTALL(TARGETS udevtrigger
RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
)
ENDIF()
//Fourth, utrace user space trace tool
IF(UTRACE_SUPPORT)
ADD_EXECUTABLE(utrace trace/trace.c)
TARGET_LINK_LIBRARIES(utrace ${ubox} ${json} ${blobmsg_json})
INSTALL(TARGETS utrace
RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
)
ADD_DEPENDENCIES(utrace syscall-names-h)
There are many contents involved in this project, please refer to the notes in the document. This part is the realization of the core basic functions of OpenWrt system. Through this part, you can clearly see the system startup process. As the content covers a wide range, the process file of analyzing the source code is posted for your reference as the source code relationship map.
Source code analysis relationship record
1, The system starts the initialization / sbin/init thread
The kernel retrieves the evn arg environment variable and executes the user state initialization thread / sbin/init;
1.1 entry main function
int main(int argc, char **argv)
(1). Initialization semaphore
ulog_open(ULOG_KMSG, LOG_DAEMON, "init");
sigaction(SIGTERM, &sa_shutdown, NULL);
sigaction(SIGUSR1, &sa_shutdown, NULL);
sigaction(SIGUSR2, &sa_shutdown, NULL);
sigaction(SIGPWR, &sa_shutdown, NULL);
(2).
early();
(3).
cmdline();
(4).
watchdog_init(1);
(5).
pid = fork();
if (!pid) {
char *kmod[] = { "/sbin/kmodloader", "/etc/modules-boot.d/", NULL };
if (debug < 3)
patch_stdio("/dev/null");
execvp(kmod[0], kmod); // The fork() subprocess executes the execvp() function
/*
* Function Description: execvp() will find the file name matching the parameter file from the directory indicated by the PATH environment variable, execute the file after finding it, and then pass the second parameter argv to the file to be executed.
* Return value: if the execution is successful, the function will not return. If the execution fails, it will directly return - 1. The failure reason is stored in errno
*
* Execute the / sbin/kmodloader function
* Execute / etc / modules boot Module initialization program under D / path,
* root@ixeRouter:~# ls /etc/modules-boot.d/
* 02-crypto-hash 30-fs-squashfs 50-usb-ohci
* 04-crypto-crc32c 30-gpio-button-hotplug 50-usb-uhci
* 09-crypto-aead 35-usb-ehci 51-usb-ohci-pci
* 09-crypto-manager 40-scsi-core 54-usb3
* 15-libphy 40-usb2 60-leds-gpio
* 15-mii 41-ata-ahci mmc
* 20-usb-core 42-usb2-pci sdhci-mt7620
*/
ERROR("Failed to start kmodloader: %m\n");
exit(EXIT_FAILURE);
}
if (pid <= 0) {
ERROR("Failed to start kmodloader instance: %m\n");
} else {//The main thread executes the following program
const struct timespec req = {0, 10 * 1000 * 1000};
int i;
for (i = 0; i < 1200; i++) {
if (waitpid(pid, NULL, WNOHANG) > 0)
break;
nanosleep(&req, NULL);
watchdog_ping();
}
}
(6).
uloop_init();
(7).
preinit();
char *init[] = { "/bin/sh", "/etc/preinit", NULL };
char *plug[] = { "/sbin/procd", "-h", "/etc/hotplug-preinit.json", NULL };
plugd_proc.pid = fork(); // fork() creates a child process
if (!plugd_proc.pid) {
execvp(plug[0], plug); // The child process executes the / sbin/procd function, and the 1# thread is created successfully
ERROR("Failed to start plugd: %m\n");
exit(EXIT_FAILURE);
}
if (plugd_proc.pid <= 0) {
ERROR("Failed to start new plugd instance: %m\n");
return;
}
uloop_process_add(&plugd_proc); // Put procd into uloop linked list
setenv("PREINIT", "1", 1);
fd = creat("/tmp/.preinit", 0600);
if (fd < 0)
ERROR("Failed to create sentinel file: %m\n");
else
close(fd);
preinit_proc.cb = spawn_procd;
preinit_proc.pid = fork(); //fork() creates a child process
if (!preinit_proc.pid) {
execvp(init[0], init); // The child process executes / bin/sh, and the thread is created successfully
ERROR("Failed to start preinit: %m\n");
exit(EXIT_FAILURE);
}
if (preinit_proc.pid <= 0) {
ERROR("Failed to start new preinit instance: %m\n");
return;
}
uloop_process_add(&preinit_proc); hold spawn_procd put to uloop In the linked list
(8).
uloop_run();
1.2 function spawn_procd entrance
static void spawn_procd(struct uloop_process *proc, int ret)
{
char *wdt_fd = watchdog_fd();
char *argv[] = { "/sbin/procd", NULL};
if (plugd_proc.pid > 0)
kill(plugd_proc.pid, SIGKILL);
unsetenv("INITRAMFS");
unsetenv("PREINIT");
unlink("/tmp/.preinit");
check_sysupgrade(); //Check for system software updates
DEBUG(2, "Exec to real procd now\n");
if (wdt_fd)
setenv("WDTFD", wdt_fd, 1);
check_dbglvl();
if (debug > 0) {
snprintf(dbg, 2, "%d", debug);
setenv("DBGLVL", dbg, 1);
}
execvp(argv[0], argv);
}
1.3 system software check and update
static void check_sysupgrade(void)
{
char *prefix = NULL, *path = NULL, *command = NULL;
size_t n;
if (chdir("/"))
return;
FILE *sysupgrade = fopen("/tmp/sysupgrade", "r");
if (!sysupgrade)
return;
n = 0;
if (getdelim(&prefix, &n, 0, sysupgrade) < 0)
goto fail;
n = 0;
if (getdelim(&path, &n, 0, sysupgrade) < 0)
goto fail;
n = 0;
if (getdelim(&command, &n, 0, sysupgrade) < 0)
goto fail;
fclose(sysupgrade);
sysupgrade_exec_upgraded(prefix, path, NULL, command, NULL); // Perform system upgrade / sbin/upgraded
while (true)
sleep(1);
fail:
fclose(sysupgrade);
free(prefix);
free(path);
free(command);
}
2, procd function analysis of process 1
2.0 procd. main function in C file
(1).
uloop_init();
(1.0)
uloop_init_pollfd()；
epoll || kqueue initialization
(1.1).
waker_init()；
waker_fd.cb = waker_consume()
uloop_fd_add(&waker_fd, ULOOP_READ)
(1.2).
uloop_setup_signals(true);
uloop_install_handler(SIGINT, uloop_handle_sigint, &old_sigint, add); //Initialization signal SIGINT
uloop_install_handler(SIGTERM, uloop_handle_sigint, &old_sigterm, add); //Initialization signal SIGTERM
uloop_install_handler(SIGCHLD, uloop_sigchld, &old_sigchld, add); //Initialization signal SIGCHLD
uloop_ignore_signal(SIGPIPE, add); //Initialization signal SIGCHLD
(2).
procd_signal()
sigaction(SIGTERM, &sa_shutdown, NULL);
sigaction(SIGINT, &sa_shutdown, NULL);
sigaction(SIGUSR1, &sa_shutdown, NULL);
sigaction(SIGUSR2, &sa_shutdown, NULL);
sigaction(SIGPWR, &sa_shutdown, NULL);
signal_shutdown(); //reboot
sigaction(SIGSEGV, &sa_crash, NULL);
sigaction(SIGBUS, &sa_crash, NULL);
do_reboot(); //reboot
sigaction(SIGHUP, &sa_dummy, NULL);
sigaction(SIGKILL, &sa_dummy, NULL);
sigaction(SIGSTOP, &sa_dummy, NULL);
signal_dummy(); //Output error message
#ifndef DISABLE_INIT
reboot(RB_DISABLE_CAD); //
#endif
(3).
if (getpid() != 1)
procd_connect_ubus(); // Initialize the ubus bus
else
procd_state_next(); //Loop state machine variables to switch program running state
void procd_state_next(void)
{
DEBUG(4, "Change state %d -> %d\n", state, state + 1);
state++;
state_enter(); //Call the state machine processing function state_enter()
}
(4).
uloop_run();
uloop_run_timeout(-1);
uloop_handle_processes(); //
uloop_run_events(next_time); // Execute RPC callback function of ubus
(5).
uloop_done();
2.1 function state_enter()
// State machine parameter definition
static int state = STATE_NONE;
enum {
STATE_NONE = 0,
STATE_EARLY,
STATE_UBUS,
STATE_INIT,
STATE_RUNNING,
STATE_SHUTDOWN,
STATE_HALT,
__STATE_MAX,
};
// Running order of state machine: state_ EARLY -> STATE_ UBUS -> STATE_ INIT -> STATE_ RUNNING;
// STATE_SHUTDOWN,STATE_HALT
static void state_enter(void)
{
char ubus_cmd[] = "/sbin/ubusd";
switch (state) {
case STATE_EARLY:
LOG("- early -\n");
watchdog_init(0); // Initialize watchdog
hotplug("/etc/hotplug.json"); // Hot plug
procd_coldplug(); // Cold plug
break;
case STATE_UBUS:
// try to reopen incase the wdt was not available before coldplug
watchdog_init(0); // Initialize watchdog
set_stdio("console"); // Set console parameters
LOG("- ubus -\n");
procd_connect_ubus(); // Connecting the ubus bus
service_start_early("ubus", ubus_cmd); // Start RPC
break;
case STATE_INIT:
LOG("- init -\n");
procd_inittab(); //Execute the script functions in the / etc/inittab file,
::sysinit:/etc/init.d/rcS S boot
::shutdown:/etc/init.d/rcS K shutdown
::askconsole:/usr/libexec/login.sh
procd_inittab_run("respawn"); //
procd_inittab_run("askconsole");
procd_inittab_run("askfirst");
procd_inittab_run("sysinit");
// switch to syslog log channel
ulog_open(ULOG_SYSLOG, LOG_DAEMON, "procd");
break;
case STATE_RUNNING:
LOG("- init complete -\n");
procd_inittab_run("respawnlate");
procd_inittab_run("askconsolelate");
break;
case STATE_SHUTDOWN:
/* Redirect output to the console for the users' benefit */
set_console(); // Redirect the console port according to the parameters passed from uboot to the / proc/cmdline file
LOG("- shutdown -\n");
procd_inittab_run("shutdown"); // Initialize shutdown callback function
sync();
break;
case STATE_HALT:
// To prevent killed processes from interrupting the sleep
signal(SIGCHLD, SIG_IGN);
LOG("- SIGTERM processes -\n");
kill(-1, SIGTERM);
sync();
sleep(1);
LOG("- SIGKILL processes -\n");
kill(-1, SIGKILL);
sync();
sleep(1);
#ifndef DISABLE_INIT
perform_halt();
#else
exit(EXIT_SUCCESS);
#endif
break;
default:
ERROR("Unhandled state %d\n", state);
return;
};
}
2.2 watchdog initialization, input parameter preinit=0
static struct uloop_timeout wdt_timeout;
static int wdt_fd = -1;
static int wdt_drv_timeout = 30; //Watchdog timeout: 30s
static int wdt_frequency = 5; //Dog feeding frequency: 5s
static bool wdt_magicclose = false;
void watchdog_init(int preinit)
{
wdt_timeout.cb = watchdog_timeout_cb;
if (watchdog_open(!preinit) < 0)
return;
LOG("- watchdog -\n");
watchdog_set_drv_timeout(); //Set watchdog time = wdt_drv_timeout=30s
watchdog_timeout_cb(&wdt_timeout);
DEBUG(4, "Opened watchdog with timeout %ds\n", watchdog_timeout(0));
}
2.3 inittab.c file procd_ Detailed explanation of inittab function
// Array handlers [] initialization parameters
static struct init_handler handlers[] = {
{
.name = "sysinit",
.cb = runrc, // Execute / etc / RC File under D path
}, {
.name = "shutdown",
.cb = runrc,
}, {
.name = "askfirst",
.cb = askfirst,
.multi = 1,
}, {
.name = "askconsole",
.cb = askconsole,
.multi = 1,
}, {
.name = "respawn",
.cb = rcrespawn,
.multi = 1,
}, {
.name = "askconsolelate",
.cb = askconsole,
.multi = 1,
}, {
.name = "respawnlate",
.cb = rcrespawn,
.multi = 1,
}
};
struct init_action {
struct list_head list;
char *id;
char *argv[MAX_ARGS];
char *line;
struct init_handler *handler;
struct uloop_process proc;
int respawn;
struct uloop_timeout tout;
};
static const char *tab = "/etc/inittab";
static char *ask = "/sbin/askfirst";
static LIST_HEAD(actions);
Entry function
void procd_inittab(void){
// Read the contents of the / etc/inittab file and put:: sysinit: / etc / init d/rcS S boot
// ::shutdown:/etc/init.d/rcS K shutdown
// ::askconsole:/usr/libexec/login.sh
// Add the contents to the handlers [] array,
while (fgets(line, LINE_LEN, fp)) {
add_action(a, tags[TAG_ACTION]) //Add item by item to
}
}
// This fork is called in the callback function_ Worker, fork other threads.
static void fork_worker(struct init_action *a)
{
pid_t p;
a->proc.pid = fork();
if (!a->proc.pid) {
p = setsid();
if (patch_stdio(a->id))
ERROR("Failed to setup i/o redirection\n");
ioctl(STDIN_FILENO, TIOCSCTTY, 1);
tcsetpgrp(STDIN_FILENO, p);
execvp(a->argv[0], a->argv); //Give * a parameter, fork sub thread
ERROR("Failed to execute %s: %m\n", a->argv[0]);
exit(-1);
}
if (a->proc.pid > 0) {
DEBUG(4, "Launched new %s action, pid=%d\n",
a->handler->name,
(int) a->proc.pid);
uloop_process_add(&a->proc); //Register * a on the ubus bus
}
}
// Callback parameter of uloop thread
struct uloop_process
{
struct list_head list;
bool pending;
uloop_process_handler cb;
pid_t pid;
};
// Global linked list of rpc interface of ubus bus
static struct list_head timeouts = LIST_HEAD_INIT(timeouts); //Timeout callback rpc interface function
static struct list_head processes = LIST_HEAD_INIT(processes); //Explicit callback rpc interface function
// Register function
int uloop_process_add(struct uloop_process *p)
{
struct uloop_process *tmp;
struct list_head *h = &processes;
if (p->pending)
return -1;
list_for_each_entry(tmp, &processes, list) {
if (tmp->pid > p->pid) {
h = &tmp->list;
break;
}
}
list_add_tail(&p->list, h); //Add * p to the processes linked list and register it with the ubus bus,
p->pending = true;
return 0;
}
3, udevtrigger device scan thread (udev)
udevtrigger. Analysis of C file and main() entry function
This section is the device file system registration process
int main(int argc, char *argv[], char *envp[])
{
openlog("udevtrigger", LOG_PID | LOG_CONS, LOG_DAEMON);
scan_subdir("/sys/bus", "/devices", false, 1);
scan_subdir("/sys/class", NULL, false, 1); // Processing devices in / sys/class directory
if (stat("/sys/class/block", &statbuf) != 0)
scan_subdir("/sys/block", NULL, true, 1); //Processing system partition information
exit:
closelog();
return 0;
}
// Recursive scan device directory
static void scan_subdir(const char *base, const char *subdir,
bool insert, int depth)
{
DIR *dir;
struct dirent *dent;
dir = opendir(base);
if (dir == NULL)
return;
for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) {
char dirname[PATH_SIZE];
if (dent->d_name[0] == '.')
continue;
strlcpy(dirname, base, sizeof(dirname));
strlcat(dirname, "/", sizeof(dirname));
strlcat(dirname, dent->d_name, sizeof(dirname));
if (insert) {
int err;
err = device_list_insert(dirname);
if (err)
continue;
}
if (subdir)
strlcat(dirname, subdir, sizeof(base));
if (depth)
scan_subdir(dirname, NULL, true, depth - 1);
}
closedir(dir);
}
// Add equipment to equipment linked list
static int device_list_insert(const char *path)
{
char devpath[PATH_SIZE];
struct stat statbuf;
dbg("add '%s'" , path);
/* we only have a device, if we have a dev and an uevent file */
if (!device_has_attribute(path, "/dev", S_IRUSR) ||
!device_has_attribute(path, "/uevent", S_IWUSR))
return -1;
strlcpy(devpath, &path[4], sizeof(devpath));
/* resolve possible link to real target */
if (lstat(path, &statbuf) < 0)
return -1;
if (S_ISLNK(statbuf.st_mode))
if (sysfs_resolve_link(devpath, sizeof(devpath)) != 0)
return -1;
trigger_uevent(devpath);
return 0;
}
// Map the device file to memory and follow the specified path
static int sysfs_resolve_link(char *devpath, size_t size)
{
char link_path[PATH_SIZE];
char link_target[PATH_SIZE];
int len;
int i;
int back;
strlcpy(link_path, "/sys", sizeof(link_path));
strlcat(link_path, devpath, sizeof(link_path));
len = readlink(link_path, link_target, sizeof(link_target) - 1);
/*
* Definition function: int readlink(const char * path, char * buf, size_t bufsiz);
* Function Description: readlink() will save the symbolic connection content of the parameter path to the memory space indicated by the parameter buf,
* The returned content does not end in NULL, but returns the number of characters of the string
* If the parameter bufsiz is less than the content length of the symbolic connection, the too long content will be truncated
* Return value: if the execution is successful, the file path string indicated by the symbolic connection is passed. If it fails, it returns - 1. The error code is stored in errno
*/
if (len <= 0)
return -1;
link_target[len] = '\0';
dbg("path link '%s' points to '%s'", devpath, link_target);
for (back = 0; strncmp(&link_target[back * 3], "../", 3) == 0; back++)
;
dbg("base '%s', tail '%s', back %i", devpath, &link_target[back * 3], back);
for (i = 0; i <= back; i++) {
char *pos = strrchr(devpath, '/');
if (pos == NULL)
return -1;
pos[0] = '\0';
}
dbg("after moving back '%s'", devpath);
strlcat(devpath, "/", size);
strlcat(devpath, &link_target[back * 3], size);
return 0;
}