SSL_accept mit flankengetriggertem, nicht blockierendem epoll gibt immer SSL_ERROR_WANT_READ zurück

Question

Wir arbeiten zu dritt daran, SSL zu einem epollbasierten Server hinzuzufügen, aber nach 3 Tagen Frust haben wir beschlossen, die Welt um Hilfe zu bitten.

Das Problem ist, dass SSL_accept() immer wieder SSL_ERROR_WANT_READ, wir epoll Veranstaltungen für EPOLLIN und EPOLLOUT für die zugehörige Buchse haben, aber selbst wenn wir ein Ereignis erhalten und erinnern an die SSL_accept es gibt nur den Mangel lesen .... so frustrierend.

Es scheint kein gutes Arbeitsbeispiel für die Verwendung von openssl mit epoll zu geben, die wir finden können, bitte senden Sie uns Codebeispiele, wenn es welche gibt, wir haben Tage mit diesem Problem verbracht und über jede SE/SO Nachricht gelesen verbrachte Stunden auf der Openssl-Mailing-Liste.

Wir haben eine Grundstruktur-Anwendung, die für das akzeptieren wie folgt aussieht:

  if (http_sock == source_fd || https_sock == source_fd) { 
       // new connection 
       req_type_t req_type = (http_sock == source_fd) ? HTTP : HTTPS; 
       int infd; 

       while (1) { 
        struct sockaddr_in in_addr; 
        socklen_t in_addr_len; 
        ZLOG_DBG(zlog_cat, "New %s request on source_fd: %d http_sock: %d https_sock: %d", req_type == HTTP ? "HTTP" : "HTTPS", source_fd, http_sock, https_sock); 
        in_addr_len = sizeof in_addr; 
        infd = accept4(source_fd, &in_addr, &in_addr_len, SOCK_NONBLOCK); 
        if (infd <= 0) 
        { 
         if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) { 
          /* We have processed all incoming 
          connections. */ 
          break; 
         } else { 
          ZLOG_ERR(zlog_cat, "ERROR: accept"); 
          break; 
         } 
        } 

        req = new_req_data(kds, req_type, &g_data, infd, &in_addr, in_addr_len, cert_q); 


        event.events = EPOLLIN | EPOLLRDHUP | EPOLLET; 

        if (req->type == HTTPS) { 
         req->c_tls.want_ssl_accept = 1; 
         event.events = EPOLLIN | EPOLLRDHUP | EPOLLOUT | EPOLLET; 
         ssl_establish(req); 
        } 
        if ((s = set_socket_blocking(req->client.fd, 1)) < 0) 
         err(EXIT_FAILURE, "[%s:%d] set_socket_blocking()", __FUNCTION__, __LINE__); 

        epoll_event_t *epoll_event = new_epoll_event_t(infd, req); 

        event.data.ptr = epoll_event; 
        // add new infd to our epoll listener 
        if ((s = epoll_ctl(efd, EPOLL_CTL_ADD, req->client.fd, &event)) == -1) 
         err(EXIT_FAILURE, "epoll_ctl(http) %s", strerror(errno)); 
       } 
       ZLOG_DBG(zlog_cat, "New %s connection from %s:%d on FD %d mac_address: %s", req->type ? "HTTPS" : "HTTP", 
        inet_ntoa(req->client.sock.sin_addr), ntohs(req->client.sock.sin_port), req->client.fd, req->device->mac_address); 
       g_data.conn_counter++; 
       continue; // continue to next accept event 

      } // end new connection 

Die ssl_establish() Methode sieht wie folgt aus:

int ssl_establish(req_data_t *req) { 
unsigned long e; 
int ssl_err, rc; 

// create openssl server context and ssl object 
if ((req->s_tls.ctx = create_context(0)) == NULL) { 
    ZLOG_ERR(zlog_cat, "create_context() %s", strerror(errno)); 
    return -1; 
} 

if ((req->s_tls.ssl = SSL_new(req->s_tls.ctx)) == NULL) { 
    ZLOG_ERR(zlog_cat, "SSL_new() %s", strerror(errno)); 
    return -1; 
} 

// Establishing SSL/TLS connections with client 
if (req->https_def_rsa != NULL && req->https_def_cert != NULL) { 
    req->c_tls.servername = req->https_def_domain; 
    req->c_tls.cert = req->https_def_cert; 
    req->c_tls.rsa_key = req->https_def_rsa; 
    req->c_tls.dest_ip = req->orig.sock.sin_addr.s_addr; 
    req->c_tls.dc_cache = req->dc_cache; 
    req->c_tls.q_entry = NULL; 

    if (req->c_tls.ctx == NULL) { 
     if ((req->c_tls.ctx = create_context(1)) == NULL) 
      ZLOG_ERR(zlog_cat, "ERROR create_context"); 
     if (configure_context(&req->c_tls)) 
      ZLOG_ERR(zlog_cat, "ERROR configure_context"); 
     if ((req->c_tls.ssl = SSL_new(req->c_tls.ctx)) == NULL) 
      ZLOG_ERR(zlog_cat, "ERROR ssl_new"); 

     // SSL_set_mode(req->c_tls.ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); // This doesn't seem to do anything 

     if (SSL_set_fd(req->c_tls.ssl, req->client.fd) == 0) 
      ZLOG_ERR(zlog_cat, "ERROR ssl_set_fd"); 
    } 
    req->client.ssl = req->c_tls.ssl; 
} 
if ((rc = SSL_accept(req->c_tls.ssl)) < 0) { 
    ssl_err = SSL_get_error(req->c_tls.ssl, rc); 

    switch (ssl_err) { 
     case SSL_ERROR_WANT_READ: 
      ZLOG_DBG(zlog_cat, "SSL_accept SSL_ERROR_WANT_READ"); 
      req->c_tls.want_ssl_accept = 1; 
      return 0; 
      break; 
     case SSL_ERROR_WANT_WRITE: 
      ZLOG_DBG(zlog_cat, "SSL_accept SSL_ERROR_WANT_WRITE"); 
      req->c_tls.want_ssl_accept = 1; 
      return 0; 
      break; 
     case SSL_ERROR_ZERO_RETURN: 
      ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_ZERO_RETURN"); 
      req->c_tls.want_ssl_accept = 0; 
      return -1; 
      break; 
     case SSL_ERROR_WANT_X509_LOOKUP: 
      ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_WANT_X509_LOOKUP"); 
      req->c_tls.want_ssl_accept = 0; 
      return -1; 
      break; 
     case SSL_ERROR_SYSCALL: 
      ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_SYSCALL"); 
      req->c_tls.want_ssl_accept = 0; 
      return -1; 
      break; 
     case SSL_ERROR_SSL: 
      ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_SSL"); 
      req->c_tls.want_ssl_accept = 0; 
      return -1; 
      break; 
    } 
} 
ZLOG_DBG(zlog_cat, "SSL_accept success"); 
req->c_tls.want_ssl_accept = 0; 
return 1; 
} 

In den pollin/POLLOUT Handler wir dies tun:

   if (req->type == HTTPS && req->c_tls.want_ssl_accept) { 
        ZLOG_DBG(zlog_cat, "ssl_establish EPOLLIN"); 
        if ((rc = ssl_establish(req)) <= 0) { 
         ZLOG_DBG(zlog_cat, "ssl_establish EPOLLIN retry..."); 
         continue; 
        } 
       } 

Unser Protokoll sieht so aus:

2016-08-03 23:26:32.224 New HTTPS connection from 192.168.1.195:52659 on FD 15 mac_address: e4:f8:9c:85:63:99 
2016-08-03 23:26:32.224 epoll_wait returned 1 
2016-08-03 23:26:32.224 epoll event number: 0 on fd: 15 
2016-08-03 23:26:32.224 Reading data from Client, source_fd: 15 source_sock->fd: 15 dest_sock->fd: -1 mac_address: e4:f8:9c:85:63:99 
2016-08-03 23:26:32.224 ssl_establish EPOLLIN 
2016-08-03 23:26:32.484 SSL_accept SSL_ERROR_WANT_READ 
2016-08-03 23:26:32.484 ssl_establish EPOLLIN retry... 
2016-08-03 23:26:32.954 New HTTPS connection from 192.168.1.195:52660 on FD 16 mac_address: e4:f8:9c:85:63:99 
2016-08-03 23:26:32.954 epoll_wait returned 1 
2016-08-03 23:26:32.954 epoll event number: 0 on fd: 16 
2016-08-03 23:26:32.954 Establishing SSL (EPOLLOUT) 
2016-08-03 23:26:32.974 SSL_accept SSL_ERROR_WANT_READ 
2016-08-03 23:26:32.974 ssl_establish EPOLLOUT retry... 

Answer 1

Okay - ich habe es endlich funktioniert. Wenn jemand darüber stolpert, verwenden Sie nicht SSL_accept().

Ich habe einige Zustand zu meinem Haupt-Struktur und die aktualisierte die ssl_establish() Methode Tracking wie folgt aussehen:

int ssl_establish(req_data_t *req, epoll_event_t *epoll_event) { 
    unsigned long e; 
    int ssl_err, rc; 
    struct epoll_event event; 
    memset(&event, 0, sizeof(struct epoll_event)); 
    event.events = req->client.events; 
    event.data.ptr = epoll_event; 

    if (!req->client.tcp_connected) { 
     struct pollfd pfd; 
     pfd.fd = req->client.fd; 
     pfd.events = POLLOUT | POLLERR; 
     int r = poll(&pfd, 1, 0); 
     if (r == 1 && pfd.revents == POLLOUT) { 
      ZLOG_DBG(zlog_cat, "tcp connected fd %d", req->client.fd); 
      req->client.tcp_connected = 1; 
      req->client.events = EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET; 
      event.events = req->client.events; 
      if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0) 
       ZLOG_ERR(zlog_cat, "ERROR: unable to modify epoll_ctl"); 
     } else { 
      ZLOG_ERR(zlog_cat, "[%d | %d] ERROR poll fd return %d revents %d", req->client.fd, req->state, r, pfd.revents); 
      return -1; 
     } 
    } 


    // create openssl server context and ssl object 
    if (!req->s_tls.ctx) 
     if ((req->s_tls.ctx = create_context(0)) == NULL) { 
      ZLOG_ERR(zlog_cat, "create_context() %s", strerror(errno)); 
      return -1; 
     } 

    if (!req->s_tls.ssl) 
     if ((req->s_tls.ssl = SSL_new(req->s_tls.ctx)) == NULL) { 
      ZLOG_ERR(zlog_cat, "SSL_new() %s", strerror(errno)); 
      return -1; 
     } 

    req->orig.ssl = req->s_tls.ssl; 
    req->proxy.ssl = req->s_tls.ssl; 

    // Establishing SSL/TLS connections with client 
    if (req->https_def_rsa != NULL && req->https_def_cert != NULL) { 
     req->c_tls.servername = req->https_def_domain; 
     req->c_tls.cert = req->https_def_cert; 
     req->c_tls.rsa_key = req->https_def_rsa; 
     req->c_tls.dest_ip = req->orig.sock.sin_addr.s_addr; 
     req->c_tls.kudoso = req->kudoso; 
     req->c_tls.dc_cache = req->dc_cache; 
     req->c_tls.q_entry = NULL; 

     if (!req->errBio) 
      req->errBio = BIO_new_fd(2, BIO_NOCLOSE); 


     if (!req->c_tls.ctx) { 
      if ((req->c_tls.ctx = create_context(1)) == NULL) 
       ZLOG_ERR(zlog_cat, "ERROR create_context"); 
      if (configure_context(&req->c_tls)) 
       ZLOG_ERR(zlog_cat, "ERROR configure_context"); 
     } 
     if (!req->c_tls.ssl) { 
      if ((req->c_tls.ssl = SSL_new(req->c_tls.ctx)) == NULL) 
       ZLOG_ERR(zlog_cat, "ERROR ssl_new"); 

      // SSL_set_mode(req->c_tls.ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); 

      if (SSL_set_fd(req->c_tls.ssl, req->client.fd) == 0) 
       ZLOG_ERR(zlog_cat, "ERROR ssl_set_fd"); 

      SSL_set_accept_state(req->c_tls.ssl); 
      req->client.ssl = req->c_tls.ssl; 
     } 
    } 
    int r = SSL_do_handshake(req->c_tls.ssl); 
    if (r == 1) { 
     req->client.ssl_connected = 1; 
     ZLOG_DBG(zlog_cat, "[%d | %d] ssl connected", req->client.fd, req->state); 
     req->client.events = EPOLLIN | EPOLLRDHUP | EPOLLET; 
     event.events = req->client.events; 
     if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0) 
      ZLOG_ERR(zlog_cat, "ERROR: unable to modify epoll_ctl"); 
     return 1; 
    } 
    int err = SSL_get_error(req->c_tls.ssl, r); 
    int oldev = req->client.events; 
    if (err == SSL_ERROR_WANT_WRITE) { 
     req->client.events |= EPOLLOUT; 
     req->client.events &= ~EPOLLIN; 
     ZLOG_DBG(zlog_cat, "do_handshake return want write set events %d", req->client.events); 
     if (oldev == req->client.events) return 0; 
    } else if (err == SSL_ERROR_WANT_READ) { 
     req->client.events |= EPOLLIN; 
     req->client.events &= ~EPOLLOUT; 
     ZLOG_DBG(zlog_cat, "do_handshake return want read set events %d", req->client.events); 
     if (oldev == req->client.events) return 0; 
    } else { 
     ZLOG_ERR(zlog_cat, "ERROR SSL_do_handshake return %d error %d errno %d msg %s", r, err, errno, strerror(errno)); 
     ERR_print_errors(req->errBio); 
     return -1; 
    } 
    event.events = req->client.events; 
    if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0) 
     ZLOG_ERR(zlog_cat, "[%d | %d] ERROR: unable to modify epoll_ctl", req->client.fd, req->state); 
    return 0; 
}