integrate the IO queueing patch I did a number of years ago..

It uses additional threads to have more IOs inflight.  This
should be useful for devices, but likely not for files.

ggated/ggated.8

@@ -37,6 +37,7 @@
 .Op Fl v
 .Op Fl a Ar address
 .Op Fl p Ar port
+.Op Fl q Ar queue_size
 .Op Fl F Ar pidfile
 .Op Fl R Ar rcvbuf
 .Op Fl S Ar sndbuf
@@ -68,6 +69,11 @@ Port on which
 .Nm
 listens for connections.
 Default is 3080.
+.It Fl q Ar queue_size
+The number of IOs to queue to the disks at once.
+The default is 20.
+If you have a large number of disks and/or you are not seeing the expected
+number of IOPS, increase this parameter.
 .It Fl F Ar pidfile
 PID file that
 .Nm

ggated/ggated.c

@@ -95,6 +95,7 @@ struct ggd_export {
 	SLIST_ENTRY(ggd_export) e_next;
 };
 
+static int niothreads = 20;
 static const char *exports_file = GGATED_EXPORT_FILE;
 static int got_sighup = 0;
 
@@ -530,6 +531,7 @@ connection_launch(struct ggd_connection *conn)
 {
 	pthread_t td;
 	int error, pid;
+	int i;
 
 	pid = fork();
 	if (pid > 0)
@@ -581,6 +583,14 @@ connection_launch(struct ggd_connection *conn)
 		g_gate_xlog("pthread_create(recv_thread): %s.",
 		    strerror(error));
 	}
+	for (i = 0; niothreads && i < niothreads - 1; i++) {
+		error = pthread_create(&td, NULL, disk_thread, conn);
+		if (error != 0) {
+			g_gate_xlog("pthread_create(disk_thread): %s.",
+			    strerror(error));
+		}
+	}
+
 	disk_thread(conn);
 }
 
@@ -1017,7 +1027,7 @@ main(int argc, char *argv[])
 
 	bindaddr = htonl(INADDR_ANY);
 	port = G_GATE_PORT;
-	while ((ch = getopt(argc, argv, "a:hnp:F:R:S:s:v")) != -1) {
+	while ((ch = getopt(argc, argv, "a:hnp:q:F:R:S:s:v")) != -1) {
 		switch (ch) {
 		case 'a':
 			bindaddr = g_gate_str2ip(optarg);
@@ -1038,6 +1048,11 @@ main(int argc, char *argv[])
 			if (port == 0 && errno != 0)
 				errx(EXIT_FAILURE, "Invalid port.");
 			break;
+		case 'q':
+			niothreads = strtol(optarg, NULL, 10);
+			if (niothreads < 0)
+				errx(EXIT_FAILURE, "Invalid queue size.");
+			break;
 		case 'R':
 			errno = 0;
 			rcvbuf = strtoul(optarg, NULL, 10);

tests/ggate_test.sh

@@ -148,7 +148,7 @@ ggate_sock_body()
 	echo $CONF >> $PLAINFILES
 	echo "0.0.0.0 RW /dev/$work" > $CONF
 
-	atf_check ggated -s local.sock -F $PIDFILE $CONF
+	atf_check ggated -q 10 -s local.sock -F $PIDFILE $CONF
 	atf_check ggatec create -u $us sock:"$(pwd)/local.sock" /dev/$work
 
 	ggate_dev=/dev/ggate${us}