linux下epoll的ET非阻塞模式如何正常关闭连接？

Question

场景描述：

1. 服务端自身FD用的是默认的LT阻塞模式，有新连接时，客户端的FD设置为非阻塞ET模式。循环调用read，每次读取3个字节，设置了EAGAIN错误处理，出现该错误返回至epoll_wait()处继续监听。
2. 客户端连接后间隔5秒，每次发送10字节数据。循环3次后调用close/shutdown关闭连接。设置了SIGINT信号捕捉，收到ctrl + c 组合键信号，也会调用close/shutdown关闭连接。

代码：

server

#include <stdio.h>
#include <ctype.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <string.h>
#include <sys/epoll.h>
#include <stdlib.h>
#include <errno.h>
#include <fcntl.h>

#define PORT 8888
#define MAX_CONNECT 10
#define IN_ET_MODE EPOLLIN | EPOLLET
#define OUT_ET_MODE EPOLLOUT | EPOLLET

#define sys_err(ret, str) \
    do                    \
    {                     \
        if (ret == -1)    \
        {                 \
            perror(str);  \
            exit(-1);     \
        }                 \
    } while (0)

int main(int argc, char const *argv[])
{
    int sfd, cfd, epfd, ret, rdlen, i, EventNum;
    char buf[BUFSIZ] = {0};
    char str[INET_ADDRSTRLEN] = {0};
    socklen_t socklen = sizeof(struct sockaddr_in);
    struct sockaddr_in srv, clt;
    struct epoll_event evt, evts[MAX_CONNECT];
    memset(&srv, 0, sizeof(srv));
    memset(&clt, 0, sizeof(clt));
    memset(&evt, 0, sizeof(evt));
    memset(&evts, 0, sizeof(evts));

    sfd = socket(AF_INET, SOCK_STREAM, 0);
    sys_err(sfd, "socket");

    // 1. 创建epoll_event句柄,句柄中含有红黑数根节点
    epfd = epoll_create(10);
    sys_err(epfd, "epoll_create");

    // 2. 把sfd添加至红黑树中
    evt.events = EPOLLIN;
    evt.data.fd = sfd;
    ret = epoll_ctl(epfd, EPOLL_CTL_ADD, sfd, &evt);
    sys_err(ret, "epoll_ctl");

    //端口复用
    int opt = 1;
    ret = setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
    sys_err(ret, "setsockopt");

    srv.sin_family = AF_INET;
    srv.sin_port = htons(PORT);
    srv.sin_addr.s_addr = htonl(INADDR_ANY);
    ret = bind(sfd, (struct sockaddr *)&srv, socklen);
    sys_err(ret, "bind");

    ret = listen(sfd, 128);
    sys_err(ret, "listen");

    while (1)
    {
        printf("epoll_wait()开始监听事件\n");
        EventNum = epoll_wait(epfd, evts, MAX_CONNECT, -1);
        sys_err(EventNum, "epoll_wait");

        for (i = 0; i < EventNum; i++)
        {

            if (evts[i].data.fd == sfd) //连接请求
            {
                printf("有连接请求事件 %d\n", EventNum);
                cfd = accept(sfd, (struct sockaddr *)&clt, &socklen);
                sys_err(cfd, "accept");
                printf("客户端 %d [%s:%u] 已连接\n",
                       cfd,
                       inet_ntop(AF_INET, &clt.sin_addr.s_addr, str, sizeof(str)),
                       ntohs(clt.sin_port));
                //非阻塞
                int flag = fcntl(cfd, F_GETFL);
                flag |= O_NONBLOCK;
                fcntl(cfd, F_SETFL, flag);
                //添加到epoll_event句柄
                evt.events = EPOLLIN | EPOLLET;
                evt.data.fd = cfd;
                ret = epoll_ctl(epfd, EPOLL_CTL_ADD, cfd, &evt);
                sys_err(ret, "epoll_ctl");
            }
            else //数据请求
            {
                printf("有数据请求事件 %d\n", EventNum);
                int sockfd = evts[i].data.fd;
                //printf("客户端 %d 数据请求:\n",sockfd);
                while (1)
                {
                    memset(buf, 0, sizeof(buf));
                    rdlen = read(sockfd, buf, 3);
                    //printf("read return %d\n",rdlen);
                    if (rdlen < 0)
                    {
                        if (errno == EAGAIN || errno == EWOULDBLOCK)
                        {
                            printf("[EAGAIN] errno = %d, return %d\n", errno, rdlen);
                            break;
                        }
                        else
                        {
                            perror("read");
                            exit(-1);
                        }
                    }
                    else if (rdlen > 0)
                    {
                        printf("[RECV] %s, %d\n", buf, rdlen);
                        int j;
                        for (j = 0; j < rdlen; ++j)
                            buf[j] = toupper(buf[j]);
                        write(sockfd, buf, rdlen);
                    }
                    else
                    {
                        ret = epoll_ctl(epfd, EPOLL_CTL_DEL, sockfd, NULL);
                        sys_err(ret, "epoll_ctl");
                        close(sockfd);
                        printf("客户端 %d 断开连接\n", sockfd);
                        break;
                    }
                }
            }
        }
    }

    return 0;
}

client

#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <stdlib.h>
#include <signal.h>

#define PORT 8888
#define sys_err(ret, str) \
    do                    \
    {                     \
        if (ret == -1)    \
        {                 \
            perror(str);  \
            exit(-1);     \
        }                 \
    } while (0)

int cfd;

void sigproc(int signo)
{
    printf("收到ctrl + C 信号，断开连接\n");
    close(cfd);
    //shutdown(cfd, SHUT_RDWR);
    exit(-1);
}
int main(int argc, char const *argv[])
{
    char buf[BUFSIZ] = {"aaaaaaaaaa"};
    struct sockaddr_in srv;
    memset(&srv, 0, sizeof(srv));

    signal(SIGINT, sigproc);

    cfd = socket(AF_INET, SOCK_STREAM, 0);

    srv.sin_family = AF_INET;
    srv.sin_port = htons(PORT);
    inet_pton(AF_INET, "127.0.0.1", &srv.sin_addr.s_addr);

    socklen_t len = sizeof(struct sockaddr_in);
    int ret = connect(cfd, (struct sockaddr *)&srv, len);
    sys_err(ret, "connect");

    int n = 3;
    while (n--)
    {
        ret = write(cfd, buf, strlen(buf));
        printf("[SEND] %s, %d\n", buf, ret);
        sleep(5);
    }
    //shutdown(cfd, SHUT_RDWR);
    close(cfd);
    return 0;
}

问题1：
以上情景，无论在客户端第一次发送数据时发送SIGINTR信号关闭连接，还是循环完3次调用的close或shutdown关闭连接，都不能正常的4次挥手关闭，都是客户端向服务端发送了RST标志，粗暴的断开连接。

问题2：
还有，如果客户端调用的是close关闭连接，服务端errno == 104错误，Connection reset by peer，相同的位置注释掉close，使用是shutdown，errno == 11，这个又是为什么？

Answer 1

https://blog.csdn.net/zgl_dm/article/details/8569857