1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

tcp_fastopen - INTEGER
	Enable TCP Fast Open feature (draft-ietf-tcpm-fastopen) to send data
	in the opening SYN packet. To use this feature, the client application
	must use sendmsg() or sendto() with MSG_FASTOPEN flag rather than
	connect() to perform a TCP handshake automatically.

	The values (bitmap) are
	1: Enables sending data in the opening SYN on the client w/ MSG_FASTOPEN.
	2: Enables TCP Fast Open on the server side, i.e., allowing data in
	   a SYN packet to be accepted and passed to the application before
	   3-way hand shake finishes.
	4: Send data in the opening SYN regardless of cookie availability and
	   without a cookie option.
	0x100: Accept SYN data w/o validating the cookie.
	0x200: Accept data-in-SYN w/o any cookie option present.
	0x400/0x800: Enable Fast Open on all listeners regardless of the
	   TCP_FASTOPEN socket option. The two different flags designate two
	   different ways of setting max_qlen without the TCP_FASTOPEN socket
	   option.

	Default: 1

	Note that the client & server side Fast Open flags (1 and 2
	respectively) must be also enabled before the rest of flags can take
	effect.

	See include/net/tcp.h and the code for more details.

1
2
3

$ sysctl -a | grep fastopen

net.ipv4.tcp_fastopen = 1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

// reference: http://blog.csdn.net/hanhuili/article/details/8540227

#include <string.h>         
#include <sys/types.h> /* See NOTES */
#include <sys/socket.h>
#include <netinet/in.h>
int main(){
    int portno = 6666;
    socklen_t clilen;
    char buffer[256];
    struct sockaddr_in serv_addr, cli_addr;
    int cfd;
    int sfd = socket(AF_INET, SOCK_STREAM, 0);   // Create socket
    
    bzero((char *) &serv_addr, sizeof(serv_addr));
    serv_addr.sin_family = AF_INET;
    serv_addr.sin_addr.s_addr = INADDR_ANY;
    serv_addr.sin_port = htons(portno);
    bind(sfd, &serv_addr,sizeof(serv_addr));      // Bind to well known address
    
    int qlen = 5;                 // Value to be chosen by application
    int err = setsockopt(sfd, IPPROTO_TCP/*SOL_TCP*/, 23/*TCP_FASTOPEN*/, &qlen, sizeof(qlen));
    
    listen(sfd,1);                // Mark socket to receive connections

	while(1){
		cfd = accept(sfd, NULL, 0);   // Accept connection on new socket

		while(1){
			int len = read(cfd,buffer,256);
			if(len)
				printf("tcp fast open: %s\n",buffer);
			else
				break;
			// read and write data on connected socket cfd
		}

		memset(buffer, 0, 256);
		close(cfd);
	}

}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h> 
#include <errno.h>
int main(){
    struct sockaddr_in serv_addr;
    struct hostent *server;

    char *data = "Hello, tcp fast open";
    int data_len = strlen(data);    
    
    int sfd = socket(AF_INET, SOCK_STREAM, 0);
    server = gethostbyname("localhost");
    
    bzero((char *) &serv_addr, sizeof(serv_addr));
    serv_addr.sin_family = AF_INET;
    bcopy((char *)server->h_addr, 
         (char *)&serv_addr.sin_addr.s_addr,
         server->h_length);
    serv_addr.sin_port = htons(6666);

// /usr/src/linux-headers-4.4.0-22/include/linux/socket.h:#define MSG_FASTOPEN	0x20000000	/* Send data in TCP SYN */

//  int len = sendto(sfd, data, data_len, 0x20000000/*MSG_FASTOPEN*/, 

    int len = sendto(sfd, data, data_len, MSG_FASTOPEN/*MSG_FASTOPEN*/, 
                (struct sockaddr *) &serv_addr, sizeof(serv_addr));
	if(errno != 0){
		printf("error: %s\n", strerror(errno));
	}
    close(sfd);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

$ sudo tcpdump -i any port 6666 -X

# 第一次 ./client.o

00:29:34.820187 IP localhost.51388 > localhost.6666: Flags [S], seq 755101042, win 43690, options [mss 65495,sackOK,TS val 17053 ecr 0,nop,wscale 7,unknown-34,nop,nop], length 0
	0x0000:  4500 0040 afef 4000 4006 8cc6 7f00 0001  E..@..@.@.......
	0x0010:  7f00 0001 c8bc 1a0a 2d01 ed72 0000 0000  ........-..r....
	0x0020:  b002 aaaa fe34 0000 0204 ffd7 0402 080a  .....4..........
	0x0030:  0000 429d 0000 0000 0103 0307 2202 0101  ..B........."...
00:29:34.820284 IP localhost.6666 > localhost.51388: Flags [S.], seq 3725111481, ack 755101043, win 43690, options [mss 65495,sackOK,TS val 17053 ecr 17053,nop,wscale 7], length 0
	0x0000:  4500 003c 0000 4000 4006 3cba 7f00 0001  E..<..@.@.<.....
	0x0010:  7f00 0001 1a0a c8bc de08 b0b9 2d01 ed73  ............-..s
	0x0020:  a012 aaaa fe30 0000 0204 ffd7 0402 080a  .....0..........
	0x0030:  0000 429d 0000 429d 0103 0307            ..B...B.....
00:29:34.820372 IP localhost.51388 > localhost.6666: Flags [P.], seq 1:21, ack 1, win 342, options [nop,nop,TS val 17053 ecr 17053], length 20
	0x0000:  4500 0048 aff0 4000 4006 8cbd 7f00 0001  E..H..@.@.......
	0x0010:  7f00 0001 c8bc 1a0a 2d01 ed73 de08 b0ba  ........-..s....
	0x0020:  8018 0156 fe3c 0000 0101 080a 0000 429d  ...V.<........B.
	0x0030:  0000 429d 4865 6c6c 6f2c 2074 6370 2066  ..B.Hello,.tcp.f
	0x0040:  6173 7420 6f70 656e                      ast.open
00:29:34.820433 IP localhost.6666 > localhost.51388: Flags [.], ack 21, win 342, options [nop,nop,TS val 17053 ecr 17053], length 0
	0x0000:  4500 0034 f227 4000 4006 4a9a 7f00 0001  E..4.'@.@.J.....
	0x0010:  7f00 0001 1a0a c8bc de08 b0ba 2d01 ed87  ............-...
	0x0020:  8010 0156 fe28 0000 0101 080a 0000 429d  ...V.(........B.
	0x0030:  0000 429d                                ..B.
00:29:34.859246 IP localhost.6666 > localhost.51388: Flags [.], ack 22, win 342, options [nop,nop,TS val 17063 ecr 17053], length 0
	0x0000:  4500 0034 f228 4000 4006 4a99 7f00 0001  E..4.(@.@.J.....
	0x0010:  7f00 0001 1a0a c8bc de08 b0ba 2d01 ed88  ............-...
	0x0020:  8010 0156 fe28 0000 0101 080a 0000 42a7  ...V.(........B.
	0x0030:  0000 429d                                ..B.

# 第二次 ./client.o

00:29:39.271936 IP localhost.51398 > localhost.6666: Flags [S], seq 2362540136, win 43690, options [mss 65495,sackOK,TS val 18166 ecr 0,nop,wscale 7,exp-tfo cookiereq], length 0
	0x0000:  4500 0040 c69e 4000 4006 7617 7f00 0001  E..@..@.@.v.....
	0x0010:  7f00 0001 c8c6 1a0a 8cd1 8068 0000 0000  ...........h....
	0x0020:  b002 aaaa fe34 0000 0204 ffd7 0402 080a  .....4..........
	0x0030:  0000 46f6 0000 0000 0103 0307 fe04 f989  ..F.............
00:29:39.271986 IP localhost.6666 > localhost.51398: Flags [S.], seq 3703577773, ack 2362540137, win 43690, options [mss 65495,sackOK,TS val 18166 ecr 18166,nop,wscale 7], length 0
	0x0000:  4500 003c 0000 4000 4006 3cba 7f00 0001  E..<..@.@.<.....
	0x0010:  7f00 0001 1a0a c8c6 dcc0 1cad 8cd1 8069  ...............i
	0x0020:  a012 aaaa fe30 0000 0204 ffd7 0402 080a  .....0..........
	0x0030:  0000 46f6 0000 46f6 0103 0307            ..F...F.....
00:29:39.272038 IP localhost.51398 > localhost.6666: Flags [P.], seq 1:21, ack 1, win 342, options [nop,nop,TS val 18166 ecr 18166], length 20
	0x0000:  4500 0048 c69f 4000 4006 760e 7f00 0001  E..H..@.@.v.....
	0x0010:  7f00 0001 c8c6 1a0a 8cd1 8069 dcc0 1cae  ...........i....
	0x0020:  8018 0156 fe3c 0000 0101 080a 0000 46f6  ...V.<........F.
	0x0030:  0000 46f6 4865 6c6c 6f2c 2074 6370 2066  ..F.Hello,.tcp.f
	0x0040:  6173 7420 6f70 656e                      ast.open
00:29:39.272072 IP localhost.6666 > localhost.51398: Flags [.], ack 21, win 342, options [nop,nop,TS val 18166 ecr 18166], length 0
	0x0000:  4500 0034 5a58 4000 4006 e269 7f00 0001  E..4ZX@.@..i....
	0x0010:  7f00 0001 1a0a c8c6 dcc0 1cae 8cd1 807d  ...............}
	0x0020:  8010 0156 fe28 0000 0101 080a 0000 46f6  ...V.(........F.
	0x0030:  0000 46f6                                ..F.
00:29:39.311280 IP localhost.6666 > localhost.51398: Flags [.], ack 22, win 342, options [nop,nop,TS val 18176 ecr 18166], length 0
	0x0000:  4500 0034 5a59 4000 4006 e268 7f00 0001  E..4ZY@.@..h....
	0x0010:  7f00 0001 1a0a c8c6 dcc0 1cae 8cd1 807e  ...............~
	0x0020:  8010 0156 fe28 0000 0101 080a 0000 4700  ...V.(........G.
	0x0030:  0000 46f6                                ..F.

1
2
3

$ sysctl -a | grep fastopen

net.ipv4.tcp_fastopen = 3

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

# 第一次，server返回cookie unknown-34 0x38af51c10bf41ca4

00:36:36.667932 IP localhost.52220 > localhost.6666: Flags [S], seq 3662514892, win 43690, options [mss 65495,sackOK,TS val 122515 ecr 0,nop,wscale 7,unknown-34,nop,nop], length 0
	0x0000:  4500 0040 545f 4000 4006 e856 7f00 0001  E..@T_@.@..V....
	0x0010:  7f00 0001 cbfc 1a0a da4d 8acc 0000 0000  .........M......
	0x0020:  b002 aaaa fe34 0000 0204 ffd7 0402 080a  .....4..........
	0x0030:  0001 de93 0000 0000 0103 0307 2202 0101  ............"...
00:36:36.667990 IP localhost.6666 > localhost.52220: Flags [S.], seq 3186866007, ack 3662514893, win 43690, options [mss 65495,sackOK,TS val 122515 ecr 122515,nop,wscale 7,unknown-34 0x38af51c10bf41ca4,nop,nop], length 0
	0x0000:  4500 0048 0000 4000 4006 3cae 7f00 0001  E..H..@.@.<.....
	0x0010:  7f00 0001 1a0a cbfc bdf3 b757 da4d 8acd  ...........W.M..
	0x0020:  d012 aaaa fe3c 0000 0204 ffd7 0402 080a  .....<..........
	0x0030:  0001 de93 0001 de93 0103 0307 220a 38af  ............".8.
	0x0040:  51c1 0bf4 1ca4 0101                      Q.......
00:36:36.668050 IP localhost.52220 > localhost.6666: Flags [P.], seq 1:21, ack 1, win 342, options [nop,nop,TS val 122515 ecr 122515], length 20
	0x0000:  4500 0048 5460 4000 4006 e84d 7f00 0001  E..HT`@.@..M....
	0x0010:  7f00 0001 cbfc 1a0a da4d 8acd bdf3 b758  .........M.....X
	0x0020:  8018 0156 fe3c 0000 0101 080a 0001 de93  ...V.<..........
	0x0030:  0001 de93 4865 6c6c 6f2c 2074 6370 2066  ....Hello,.tcp.f
	0x0040:  6173 7420 6f70 656e                      ast.open
00:36:36.668109 IP localhost.6666 > localhost.52220: Flags [.], ack 21, win 342, options [nop,nop,TS val 122515 ecr 122515], length 0
	0x0000:  4500 0034 69cb 4000 4006 d2f6 7f00 0001  E..4i.@.@.......
	0x0010:  7f00 0001 1a0a cbfc bdf3 b758 da4d 8ae1  ...........X.M..
	0x0020:  8010 0156 fe28 0000 0101 080a 0001 de93  ...V.(..........
	0x0030:  0001 de93                                ....
00:36:36.707264 IP localhost.6666 > localhost.52220: Flags [.], ack 22, win 342, options [nop,nop,TS val 122525 ecr 122515], length 0
	0x0000:  4500 0034 69cc 4000 4006 d2f5 7f00 0001  E..4i.@.@.......
	0x0010:  7f00 0001 1a0a cbfc bdf3 b758 da4d 8ae2  ...........X.M..
	0x0020:  8010 0156 fe28 0000 0101 080a 0001 de9d  ...V.(..........
	0x0030:  0001 de93                                ....


# 第二次，client发送请求时，将cookie写在syn包中，同时带上发送的数据；server端校验后(kernel和tcp/ip协议栈做校验)后返回成功，如此在3次握手中节省了一次rtt时间

00:36:38.744954 IP localhost.52226 > localhost.6666: Flags [S], seq 1820632025:1820632045, win 43690, options [mss 65495,sackOK,TS val 123034 ecr 0,nop,wscale 7,unknown-34 0x38af51c10bf41ca4,nop,nop], length 20
	0x0000:  4500 005c 4343 4000 4006 f956 7f00 0001  E..\CC@.@..V....
	0x0010:  7f00 0001 cc02 1a0a 6c84 a3d9 0000 0000  ........l.......
	0x0020:  d002 aaaa fe50 0000 0204 ffd7 0402 080a  .....P..........
	0x0030:  0001 e09a 0000 0000 0103 0307 220a 38af  ............".8.
	0x0040:  51c1 0bf4 1ca4 0101 4865 6c6c 6f2c 2074  Q.......Hello,.t
	0x0050:  6370 2066 6173 7420 6f70 656e            cp.fast.open
00:36:38.745022 IP localhost.6666 > localhost.52226: Flags [S.], seq 3848342665, ack 1820632046, win 43690, options [mss 65495,sackOK,TS val 123034 ecr 123034,nop,wscale 7], length 0
	0x0000:  4500 003c 0000 4000 4006 3cba 7f00 0001  E..<..@.@.<.....
	0x0010:  7f00 0001 1a0a cc02 e561 0c89 6c84 a3ee  .........a..l...
	0x0020:  a012 aaaa fe30 0000 0204 ffd7 0402 080a  .....0..........
	0x0030:  0001 e09a 0001 e09a 0103 0307            ............
00:36:38.745072 IP localhost.52226 > localhost.6666: Flags [.], ack 1, win 342, options [nop,nop,TS val 123034 ecr 123034], length 0
	0x0000:  4500 0034 4344 4000 4006 f97d 7f00 0001  E..4CD@.@..}....
	0x0010:  7f00 0001 cc02 1a0a 6c84 a3ee e561 0c8a  ........l....a..
	0x0020:  8010 0156 fe28 0000 0101 080a 0001 e09a  ...V.(..........
	0x0030:  0001 e09a                                ....
00:36:38.745127 IP localhost.52226 > localhost.6666: Flags [F.], seq 1, ack 1, win 342, options [nop,nop,TS val 123034 ecr 123034], length 0
	0x0000:  4500 0034 4345 4000 4006 f97c 7f00 0001  E..4CE@.@..|....
	0x0010:  7f00 0001 cc02 1a0a 6c84 a3ee e561 0c8a  ........l....a..
	0x0020:  8011 0156 fe28 0000 0101 080a 0001 e09a  ...V.(..........
	0x0030:  0001 e09a                                ....
00:36:38.747232 IP localhost.6666 > localhost.52226: Flags [.], ack 2, win 342, options [nop,nop,TS val 123035 ecr 123034], length 0
	0x0000:  4500 0034 ec10 4000 4006 50b1 7f00 0001  E..4..@.@.P.....
	0x0010:  7f00 0001 1a0a cc02 e561 0c8a 6c84 a3ef  .........a..l...
	0x0020:  8010 0156 fe28 0000 0101 080a 0001 e09b  ...V.(..........
	0x0030:  0001 e09a                                ....

1
2
3
4
5
6
7
8
9

	The values (bitmap) are
	1: Enables sending data in the opening SYN on the client w/ MSG_FASTOPEN.
		使能client端的TFO特性
	2: Enables TCP Fast Open on the server side, i.e., allowing data in
	   a SYN packet to be accepted and passed to the application before
	   3-way hand shake finishes.
		使能server端的TFO特性
	4: Send data in the opening SYN regardless of cookie availability and
	   without a cookie option.