1, Structure of sds
Definition of sds
sds.h
//Defines a char pointer
typedef char *sds;
/* Note: sdshdr5 is never used, we just access the flags byte directly.
* However is here to document the layout of type 5 SDS strings. */
struct __attribute__ ((__packed__)) sdshdr5 {
unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
char buf[];
};
//lsb stands for significant bit,
//__ attribute__ ((_packed_)) means that the structure is manually aligned.
struct __attribute__ ((__packed__)) sdshdr8 {
//Length of buf already used
uint8_t len; /* used */
//The length allocated by buf is equal to the total length of buf [] - 1, because buf has a terminator including a / 0
uint8_t alloc; /* excluding the header and null terminator */
//There are only 3 significant bits, because the type representation is 0 to 4, and 5 bits of all 8-bit flags are not used
unsigned char flags; /* 3 lsb of type, 5 unused bits */
//The actual string exists here
char buf[];
};
// With the above changes, only len and alloc are different in length
struct __attribute__ ((__packed__)) sdshdr16 {
uint16_t len; /* used */
uint16_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
// With the above changes, only len and alloc are different in length
struct __attribute__ ((__packed__)) sdshdr32 {
uint32_t len; /* used */
uint32_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
// With the above changes, only len and alloc are different in length
struct __attribute__ ((__packed__)) sdshdr64 {
uint64_t len; /* used */
uint64_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
It can be seen that the general structure of sds is the same under different lengths, and the only different sdshdr5 has not been used.
Initialization of sds
/* Create a new sds string with the content specified by the 'init' pointer
* and 'initlen'.
* If NULL is used for 'init' the string is initialized with zero bytes.
* If SDS_NOINIT is used, the buffer is left uninitialized;
*
* The string is always null-termined (all the sds strings are, always) so
* even if you create an sds string with:
*
* mystring = sdsnewlen("abc",3);
*
* You can print the string with printf() as there is an implicit \0 at the
* end of the string. However the string is binary safe and can contain
* \0 characters in the middle, as the length is stored in the sds header. */
// Because sds specifies the length of the whole character in the front, even if the \ 0 terminator appears in the middle, it can be compiled normally, without worrying about decoding failure due to the occurrence of the terminator / 0.
sds sdsnewlen(const void *init, size_t initlen) {
// This pointer will point to the beginning of the entire sds
void *sh;
// sds is actually a pointer,
// But s points to where the entire struct buf starts
sds s;
//Return different types of sds according to different lengths
char type = sdsReqType(initlen);
/* Empty strings are usually created in order to append. Use type 8
* since type 5 is not good at this. */
// Empty string is often used for append. Here you can see that sds type 5 is replaced by sds type 8
if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8;
//Get the length of struct
int hdrlen = sdsHdrSize(type);
// flag pointer, which is used to indicate the type of sds
unsigned char *fp; /* flags pointer. */
//Allocate space here + 1 is to allocate an end symbol
sh = s_malloc(hdrlen+initlen+1);
// Here, sh points to the memory address just allocated
if (sh == NULL) return NULL;
// Determine whether it is the init phase
if (init==SDS_NOINIT)
init = NULL;
// If it is not the init stage, clear 0
else if (!init)
//If init is not empty
// Set all memory to 0
memset(sh, 0, hdrlen+initlen+1);
//s points to the address at the beginning of the string. hdrlen can read sds head incorrectly
s = (char*)sh+hdrlen;
//Because we can see that the order of addresses is len, alloc, flag and buf. At present, s points to buf,
//Then go back 1 bit and fp just points to the address corresponding to flag.
fp = ((unsigned char*)s)-1;
//Here is the basis
switch(type) {
case SDS_TYPE_5: {
*fp = type | (initlen << SDS_TYPE_BITS);
break;
}
case SDS_TYPE_8: {
//The concatenation method is used here to assign the variable sh to struct sdshdr
SDS_HDR_VAR(8,s);
//Here is the initialization length. There's nothing to say here
sh->len = initlen;
sh->alloc = initlen;
//The address corresponding to fp is assigned the corresponding type, and the value of type is 1-4
*fp = type;
break;
}
case SDS_TYPE_16: {
SDS_HDR_VAR(16,s);
sh->len = initlen;
sh->alloc = initlen;
*fp = type;
break;
}
case SDS_TYPE_32: {
SDS_HDR_VAR(32,s);
sh->len = initlen;
sh->alloc = initlen;
*fp = type;
break;
}
case SDS_TYPE_64: {
SDS_HDR_VAR(64,s);
sh->len = initlen;
sh->alloc = initlen;
*fp = type;
break;
}
}
//If neither is empty, the corresponding string init is assigned to s
if (initlen && init)
//Here is the initialization of buf assignment
memcpy(s, init, initlen);
// Assign a terminator
s[initlen] = '\0';
return s;
}
// The following methods are the tools and methods used in init above
// Return different sds types according to different string size s
static inline char sdsReqType(size_t string_size) {
if (string_size < 1<<5)
return SDS_TYPE_5;
if (string_size < 1<<8)
return SDS_TYPE_8;
if (string_size < 1<<16)
return SDS_TYPE_16;
//This should be the reason for considering the 32-bit system
#if (LONG_MAX == LLONG_MAX)
if (string_size < 1ll<<32)
return SDS_TYPE_32;
return SDS_TYPE_64;
#else
return SDS_TYPE_32;
#endif
}
// Return the length of struct according to different types
static inline int sdsHdrSize(char type) {
//SDS_TYPE_MASK is used to clear unnecessary bits of SDS_TYPE_MASK is 00000111
switch(type&SDS_TYPE_MASK) {
case SDS_TYPE_5:
return sizeof(struct sdshdr5);
case SDS_TYPE_8:
return sizeof(struct sdshdr8);
case SDS_TYPE_16:
return sizeof(struct sdshdr16);
case SDS_TYPE_32:
return sizeof(struct sdshdr32);
case SDS_TYPE_64:
return sizeof(struct sdshdr64);
}
return 0;
}
#define SDS_TYPE_5 0
#define SDS_TYPE_8 1
#define SDS_TYPE_16 2
#define SDS_TYPE_32 3
#define SDS_TYPE_64 4
#define SDS_TYPE_MASK 7
#define SDS_TYPE_BITS 3
// You can see that s originally points to the position of buf, and subtracting the length of struct is exactly the starting position
// Then sh points to the beginning of the whole sds
// T is a placeholder that returns different types of struct s
#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (void*)((s)-(sizeof(struct sdshdr##T)));
//The following is to directly return the corresponding struct
#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
//This type of can be ignored because it is not used
#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
sds diagram
image
sds expansion
/* Enlarge the free space at the end of the sds string so that the caller
* is sure that after calling this function can overwrite up to addlen
* bytes after the end of the string, plus one more byte for nul term.
*
* Note: this does not change the *length* of the sds string as returned
* by sdslen(), but only the free buffer space we have. */
// To expand the capacity of sds, here is a point. If the sds itself has remaining space, the more allocated space is equal to addlen leftlen
sds sdsMakeRoomFor(sds s, size_t addlen) {
void *sh, *newsh;
//Get remaining free space
size_t avail = sdsavail(s);
size_t len, newlen;
//In the above diagram, the pointer of s is in the middle of this space, so - 1 just points to flag
char type,oldtype = s[-1] & SDS_TYPE_MASK;
int hdrlen;
/* Return ASAP if there is enough space left. */
//If the available space is greater than the length that needs to be increased, it is returned directly
if (avail >= addlen) return s;
//len used length
len = sdslen(s);
//sh returns to the starting position of the sds.
sh = (char*)s-sdsHdrSize(oldtype);
// newlen represents the minimum required length
newlen = (len+addlen);
if (newlen < SDS_MAX_PREALLOC)
//However, twice the length will be allocated if the space is less than the maximum threshold
newlen *= 2;
else
newlen += SDS_MAX_PREALLOC;
//Gets the type of the new length
type = sdsReqType(newlen);
/* Don't use type 5: the user is appending to the string and type 5 is
* not able to remember empty space, so sdsMakeRoomFor() must be called
* at every appending operation. */
if (type == SDS_TYPE_5) type = SDS_TYPE_8;
hdrlen = sdsHdrSize(type);
if (oldtype==type) {
//sh is the start address. Based on the start address, more space is allocated,
// Logic is like the initialization part. hdrlen is the length of head, that is, the size of struct itself
// The following newlen is the buf size, and + 1 is the end symbol
// sds can usually be printed directly
newsh = s_realloc(sh, hdrlen+newlen+1);
if (newsh == NULL) return NULL;
//s continues to point to the middle position
s = (char*)newsh+hdrlen;
} else {
/* Since the header size changes, need to move the string forward,
* and can't use realloc */
//If the type changes, the address content cannot be reused, so find a new space.
newsh = s_malloc(hdrlen+newlen+1);
if (newsh == NULL) return NULL;
//Copy the original str to the new sds,
//newsh+hdrlen is equal to the beginning of the sds buf address
//s location of original buf
//len+1 copies the ending symbol
memcpy((char*)newsh+hdrlen, s, len+1);
//Free up front memory space
s_free(sh);
//Adjust the starting position of s, that is, the address space points to the starting position of the new buf
s = (char*)newsh+hdrlen;
//-1 is just at the position of flag
s[-1] = type;
//Assign the value of len
sdssetlen(s, len);
}
//Assign the value of alloc
sdssetalloc(s, newlen);
//Return new sds
return s;
}
// Set len
static inline void sdssetlen(sds s, size_t newlen) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
{
unsigned char *fp = ((unsigned char*)s)-1;
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
}
break;
case SDS_TYPE_8:
SDS_HDR(8,s)->len = newlen;
break;
case SDS_TYPE_16:
SDS_HDR(16,s)->len = newlen;
break;
case SDS_TYPE_32:
SDS_HDR(32,s)->len = newlen;
break;
case SDS_TYPE_64:
SDS_HDR(64,s)->len = newlen;
break;
}
}
// Gets the available length of the current sds.
static inline size_t sdsavail(const sds s) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5: {
return 0;
}
case SDS_TYPE_8: {
SDS_HDR_VAR(8,s);
return sh->alloc - sh->len;
}
case SDS_TYPE_16: {
SDS_HDR_VAR(16,s);
return sh->alloc - sh->len;
}
case SDS_TYPE_32: {
SDS_HDR_VAR(32,s);
return sh->alloc - sh->len;
}
case SDS_TYPE_64: {
SDS_HDR_VAR(64,s);
return sh->alloc - sh->len;
}
}
return 0;
}
/* sdsalloc() = sdsavail() + sdslen() */
// Gets the length of alloc
static inline size_t sdsalloc(const sds s) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
return SDS_TYPE_5_LEN(flags);
case SDS_TYPE_8:
return SDS_HDR(8,s)->alloc;
case SDS_TYPE_16:
return SDS_HDR(16,s)->alloc;
case SDS_TYPE_32:
return SDS_HDR(32,s)->alloc;
case SDS_TYPE_64:
return SDS_HDR(64,s)->alloc;
}
return 0;
}
// Set alloc value
static inline void sdssetalloc(sds s, size_t newlen) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
/* Nothing to do, this type has no total allocation info. */
break;
case SDS_TYPE_8:
SDS_HDR(8,s)->alloc = newlen;
break;
case SDS_TYPE_16:
SDS_HDR(16,s)->alloc = newlen;
break;
case SDS_TYPE_32:
SDS_HDR(32,s)->alloc = newlen;
break;
case SDS_TYPE_64:
SDS_HDR(64,s)->alloc = newlen;
break;
}
}
The above mainly talks about how to expand the capacity of sds. We can see that the biggest feature of sds is that it can pre allocate memory and is very efficient in capacity expansion. Copy without copying
Why use memory misalignment (very important. This is the only point about sds in the whole network)
Before looking at the following, we should first have a basic concept of memory alignment. Why memory alignment is needed here is related to the working principle of cpu. You can baidu cpu working principle, which is mainly related to the mapping relationship between memory address and register, but there are two theorems that can be understood here.
-
A 32-bit cpu can read 32-bit data from memory every cycle.
-
For this reason and register reasons, the address read by the cpu each time is a multiple of 4. For example, if we want to read the data with a length of 4 on address 2, it takes two cycles. The cpu first has to read the data from address 0-3, and then from address 3-7. Here we can see the role of memory alignment. So the question is, why should a redis author, who uses the most memory and cpu, use a non aligned sds? The reason is that the structure of sds is doomed to be in a non aligned state. Please see the figure below. In the aligned state, how does our structure behave in memory.
image
It can be seen that under different types of sds, the number of pad bits is also different, so it is impossible for us to access the flag from the sds pointer without knowing the type. Then some students have to ask again. Isn't it OK to remove the structure of sdshdr8? Theoretically, this will not sacrifice too much memory and ensure the performance, But this is only the structure under the 32-bit system. If it is on the 64 bit system, it may be another structure. OK, then some students have to say whether we can put the pointer at the beginning of the flag. The answer is no, 1. In this way, we can't be perfectly compatible with string and 2. In this way, we will also introduce various types of judgment and adjustment, so redis finally uses the scheme of memory misalignment.
summary
-
It can be seen that sds is binary safe compared with ordinary c language string, because string has no length mark. When there are multiple end symbols in a section of address, string cannot access the content after the first / 0, and the structure of sds makes up for this deficiency.
-
sds is also unmatched by String for dynamic capacity expansion. For example, the structure of sds allows sds to pre allocate memory, and even on the basis of the original memory use on the basis of the original length type unchanged. To achieve dynamic capacity expansion, the above code is also very clear. What are the benefits of this? If a String is very long, if you want to perform an append operation based on the original String, you need to copy the content to the new address, which is a very performance-consuming thing, and sds just solves this problem, especially when converting network io data into specific command operations, You should often append the String. Therefore, sds structure is very suitable for redis.
-
And sds can also be directly used as a string. The clever use of pointers also makes sds perfectly compatible with strings. As for why we should talk about sds, because the most basic data in redis is sds in addition to the dictionary, so before we talk about how redis converts network messages into specific executed commands, we must first be familiar with the structure of sds.
Author: lazy programmer - Xiao Peng
Original link: https://blog.csdn.net/qq_33361976/article/details/109014012
Author: stay up late without overtime
Link: https://www.jianshu.com/p/cf88fc1b8878
Source: Jianshu
The copyright belongs to the author. For commercial reprint, please contact the author for authorization, and for non-commercial reprint, please indicate the source.