attempt to fix ggatessh to work with libssh2's broken event system...

This I believe fixes problems with it.  First off, is to call an
internal function to make sure to process any packets that are
pending despite there being no outstanding actions...

The second part is dealing w/ the issue that one channel may read
data for a previously processes channel...  There isn't a good way
to get what channels have pending data, instead, use a custom wrapper
around recv, and if ANY data was read/returned, process all the pending
requests again.  This will do a bit of extra work, but it a lot more
simple than fixing libssh2 to be sane...

ggatessh/ggatessh.c

@@ -60,6 +60,7 @@ static enum { UNSET, CREATE, DESTROY, LIST, RESCUE } action = UNSET;
 
 struct ggs_connection {
 	int			c_fd;
+	int			*c_didwork;	/* allocated memory */
 	LIBSSH2_SESSION		*c_session;
 	LIBSSH2_SFTP		*c_sftp_session;
 	LIBSSH2_SFTP_HANDLE	*c_handle;
@@ -221,6 +222,7 @@ make_connection(void)
 	LIBSSH2_SFTP *sftp_session;
 	LIBSSH2_SFTP_HANDLE *handle;
 	char *tmp;
+	int *didworkp;
 	int sockfd;
 	int rc;
 
@@ -232,8 +234,12 @@ make_connection(void)
 		g_gate_xlog("tcp_connect: %s.", strerror(errno));
 	}
 
+	didworkp = malloc(sizeof *didworkp);
+	if (didworkp == NULL)
+		g_gate_xlog("malloc failed.");
 
-	session = libssh2_session_init();
+	/* session = libssh2_session_init(); */
+	session = libssh2_session_init_ex(NULL, NULL, NULL, didworkp);
 	if (session == NULL)
 		libssh2_errorx(session, "libssh2_session_init");
 
@@ -287,6 +293,7 @@ make_connection(void)
 
 	return (struct ggs_connection){
 		.c_fd = sockfd,
+		.c_didwork = didworkp,
 		.c_session = session,
 		.c_sftp_session = sftp_session,
 		.c_handle = handle,
@@ -598,6 +605,44 @@ completeio:
 	return didwork;
 }
 
+/*
+ * XXX - expose this, this is to try to silence the loop that can
+ * happen when there is data waiting (oob or window info), but no
+ * outstanding requests are pending.
+ */
+int _libssh2_transport_read(LIBSSH2_SESSION *session);
+
+/*
+ * libssh2 does not have a good way to handle detection when it's
+ * truly time to sleep.  Writing is an easy case, as if there's space
+ * to write, and the _BLOCK_OUTBOUND flag is set, select will return
+ * and the data will get processed.
+ *
+ * In the case of reading, it is more complicated, as a later request
+ * could read data for a previous request, and there is no known way
+ * to know when this is done.  The solution I came up with is to wrap
+ * the recv function, and continue to iterate through the pending
+ * requests until no more data is read.
+ */
+
+static ssize_t
+ggatessh_recv_libssh2_hack(libssh2_socket_t fd, void *buf,
+    size_t len, int recv_flags, void **abstract)
+{
+	int *didworkp = *abstract;
+	ssize_t ret;
+
+	ret = recv(fd, buf, len, recv_flags);
+
+	if (ret > 0)
+		*didworkp = 1;
+
+	if (ret == -1 && errno == EAGAIN)
+		return -EAGAIN;
+
+	return ret;
+}
+
 /*
  * sftp session management is a bit tricky.
  * if there is an entry in sessioncache, use that one.
@@ -614,6 +659,7 @@ proc_thread(void *arg __unused)
 	struct ggs_sess_cache *gsc, *gsc_pending;
 	struct ggs_req *greq;
 	LIBSSH2_SESSION *session;
+	int *didworkp;	/* was any reads done, rescan work */
 	fd_set fdread;
 	fd_set fdwrite;
 	fd_set fdexcep;
@@ -622,7 +668,6 @@ proc_thread(void *arg __unused)
 	int error;
 	int dir;
 	int rc;
-	int didwork;	/* did libssh2 do any work? */
 
 	g_gate_log(LOG_NOTICE, "%s: started!", __func__);
 
@@ -632,6 +677,7 @@ proc_thread(void *arg __unused)
 	fcntl(popfd, F_SETFL, O_NONBLOCK);
 
 	sockfd = start_conn.c_fd;
+	didworkp = start_conn.c_didwork;
 	session = start_conn.c_session;
 
 	gsc = malloc(sizeof *gsc);
@@ -643,14 +689,16 @@ proc_thread(void *arg __unused)
 	gsc = NULL;
 	gsc_pending = NULL;
 
-	didwork = 0;
+	*didworkp = 0;
 
 	libssh2_session_set_blocking(session, 0);
+	libssh2_session_callback_set(session, LIBSSH2_CALLBACK_RECV,
+	    ggatessh_recv_libssh2_hack);
 
 	for (;;) {
 		//g_gate_log(LOG_DEBUG, "looping");
 
-		if (!didwork) {
+		if (!*didworkp) {
 			/* setup polling loop */
 			maxfd = -1;
 			FD_ZERO(&fdread);
@@ -696,10 +744,12 @@ proc_thread(void *arg __unused)
 			}
 		}
 
-		didwork = 0;
+		_libssh2_transport_read(session);
+
+		*didworkp = 0;
 
 		/* process pending, so any completed can be reused */
-		didwork |= process_pending(&req_pending, &session_cache);
+		process_pending(&req_pending, &session_cache);
 
 		if (FD_ISSET(popfd, &fdread)) {
 			/* read off the tokens */
@@ -767,13 +817,13 @@ procreq:
 		if (gsc_pending != NULL) {
 			/* we are creating a new session */
 			if (gsc_pending->sc_session == NULL) {
-				didwork = 1;
+				//didwork = 1;
 				gsc_pending->sc_session =
 				    libssh2_sftp_init(session);
 			}
 
 			if (gsc_pending->sc_session != NULL) {
-				didwork = 1;
+				//didwork = 1;
 				gsc_pending->sc_handle = libssh2_sftp_open(
 				    gsc_pending->sc_session, "fstest/data.img",
 				    get_open_flags(), 0);
@@ -789,13 +839,13 @@ procreq:
 				TAILQ_INSERT_HEAD(&session_cache, gsc_pending,
 				    sc_next);
 				gsc_pending = NULL;
-				didwork = 1;
+				//didwork = 1;
 				goto procreq;
 			}
 		}
 
 		/* kick of any queued requests from above */
-		didwork |= process_pending(&req_pending, &session_cache);
+		process_pending(&req_pending, &session_cache);
 	}
 
 	g_gate_log(LOG_DEBUG, "%s: Died.", __func__);