loader: open file list should be dynamic

Summary:
Open file list is currently created as statically allocated array (64 items).
Once this array is filled up, loader will not be able to operate with files.
In most cases, this mechanism is good enough, but the problem appears, when
we have many disks with zfs pool(s). In current loader implementation, all
discovered zfs pool configurations are kept in memory and disk devices open -
consuming the open file array. Rewrite the open file mechanism to use
dynamically allocated list.

Reviewed by:	imp
MFC after:	2 weeks
Differential Revision: https://reviews.freebsd.org/D31364

13 files changed, 227 insertions, 115 deletions

diff --git a/stand/libsa/close.c b/stand/libsa/close.c
index d8f62fef6198..a177eb85eb3a 100644
--- a/stand/libsa/close.c
+++ b/stand/libsa/close.c
@@ -68,23 +68,38 @@ __FBSDID("$FreeBSD$");
 int
 close(int fd)
 {
-	struct open_file *f = &files[fd];
+	struct open_file *f, *last;
 	int err1 = 0, err2 = 0;
 
-	if ((unsigned)fd >= SOPEN_MAX || f->f_flags == 0) {
+	f = fd2open_file(fd);
+	if (f == NULL) {
 		errno = EBADF;
 		return (-1);
 	}
 	free(f->f_rabuf);
 	f->f_rabuf = NULL;
 
-	if (!(f->f_flags & F_RAW) && f->f_ops)
-		err1 = (f->f_ops->fo_close)(f);
-	if (!(f->f_flags & F_NODEV) && f->f_dev)
-		err2 = (f->f_dev->dv_close)(f);
-	if (f->f_devdata != NULL)
-		devclose(f);
-	f->f_flags = 0;
+	if (f->f_flags != 0) {
+		if (!(f->f_flags & F_RAW) && f->f_ops)
+			err1 = (f->f_ops->fo_close)(f);
+		if (!(f->f_flags & F_NODEV) && f->f_dev)
+			err2 = (f->f_dev->dv_close)(f);
+		if (f->f_devdata != NULL)
+			devclose(f);
+		f->f_flags = 0;
+	} else {
+		/* Attempt to close already closed file. */
+		err1 = EBADF;
+	}
+
+	/* free unused entries from tail. */
+	TAILQ_FOREACH_REVERSE_SAFE(last, &files, file_list, f_link, f) {
+		if (last->f_flags != 0)
+			break;
+		TAILQ_REMOVE(&files, last, f_link);
+		free(last);
+	}
+
 	if (err1) {
 		errno = err1;
 		return (-1);
diff --git a/stand/libsa/closeall.c b/stand/libsa/closeall.c
index 9693130ebf9b..92c6955cf5da 100644
--- a/stand/libsa/closeall.c
+++ b/stand/libsa/closeall.c
@@ -1,11 +1,7 @@
-/*	$NetBSD: closeall.c,v 1.1 1996/01/13 22:25:36 leo Exp $	*/
-
 /*-
- * Copyright (c) 1993
- *	The Regents of the University of California.  All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
- * This code is derived from software contributed to Berkeley by
- * The Mach Operating System project at Carnegie-Mellon University.
+ * Copyright (c) 2021 Toomas Soome <tsoome@me.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -15,14 +11,11 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
  *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
@@ -31,33 +24,6 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	@(#)close.c	8.1 (Berkeley) 6/11/93
- *  
- *
- * Copyright (c) 1989, 1990, 1991 Carnegie Mellon University
- * All Rights Reserved.
- *
- * Author: Alessandro Forin
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie the
- * rights to redistribute these changes.
  */
 
 #include <sys/cdefs.h>
@@ -66,11 +32,17 @@ __FBSDID("$FreeBSD$");
 #include "stand.h"
 
 void
-closeall()
+closeall(void)
 {
-	int i;
+	struct open_file *f;
 
-        for (i = 0; i < SOPEN_MAX; i++)
-            if (files[i].f_flags != 0)
-                (void)close(i);
+	/*
+	 * Pick up last entry and close it, this will also trigger
+	 * the removal of this entry, and we end up with empty list.
+	 */
+	while ((f = TAILQ_LAST(&files, file_list)) != NULL) {
+		(void)close(f->f_id);
+	}
+	/* reset errno from close() */
+	errno = 0;
 }
diff --git a/stand/libsa/fstat.c b/stand/libsa/fstat.c
index cb2df8cbf0ac..47893416e0bf 100644
--- a/stand/libsa/fstat.c
+++ b/stand/libsa/fstat.c
@@ -37,13 +37,12 @@ __FBSDID("$FreeBSD$");
 #include "stand.h"
 
 int
-fstat(fd, sb)
-	int fd;
-	struct stat *sb;
+fstat(int fd, struct stat *sb)
 {
-	struct open_file *f = &files[fd];
+	struct open_file *f;
 
-	if ((unsigned)fd >= SOPEN_MAX || f->f_flags == 0) {
+	f = fd2open_file(fd);
+	if (f == NULL || f->f_flags == 0) {
 		errno = EBADF;
 		return (-1);
 	}
diff --git a/stand/libsa/ioctl.c b/stand/libsa/ioctl.c
index 807b308e5e50..7363236ada0e 100644
--- a/stand/libsa/ioctl.c
+++ b/stand/libsa/ioctl.c
@@ -32,30 +32,30 @@
  * SUCH DAMAGE.
  *
  *	@(#)ioctl.c	8.1 (Berkeley) 6/11/93
- *  
+ *
  *
  * Copyright (c) 1989, 1990, 1991 Carnegie Mellon University
  * All Rights Reserved.
  *
  * Author: Alessandro Forin
- * 
+ *
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
- * 
+ *
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
+ *
  * Carnegie Mellon requests users of this software to return to
- * 
+ *
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
- * 
+ *
  * any improvements or extensions that they make and grant Carnegie the
  * rights to redistribute these changes.
  */
@@ -66,14 +66,12 @@ __FBSDID("$FreeBSD$");
 #include "stand.h"
 
 int
-ioctl(fd, cmd, arg)
-	int fd;
-	u_long cmd;
-	char *arg;
+ioctl(int fd, u_long cmd, char *arg)
 {
-	struct open_file *f = &files[fd];
+	struct open_file *f;
 
-	if ((unsigned)fd >= SOPEN_MAX || f->f_flags == 0) {
+	f = fd2open_file(fd);
+	if (f == NULL || f->f_flags == 0) {
 		errno = EBADF;
 		return (-1);
 	}
diff --git a/stand/libsa/iodesc.h b/stand/libsa/iodesc.h
index 37ae044af9d9..199fbcf74e98 100644
--- a/stand/libsa/iodesc.h
+++ b/stand/libsa/iodesc.h
@@ -47,6 +47,8 @@ struct iodesc {
 	u_long	xid;			/* transaction identification */
 	u_char	myea[6];		/* my ethernet address */
 	struct netif *io_netif;
+	int io_id;			/* descriptor id */
+	TAILQ_ENTRY(iodesc) io_link;	/* next entry in list */
 };
 
 #endif /* __SYS_LIBNETBOOT_IODESC_H */
diff --git a/stand/libsa/lseek.c b/stand/libsa/lseek.c
index 1a39eb31421b..c4ca97b148d6 100644
--- a/stand/libsa/lseek.c
+++ b/stand/libsa/lseek.c
@@ -69,9 +69,10 @@ off_t
 lseek(int fd, off_t offset, int where)
 {
 	off_t bufpos, filepos, target;
-	struct open_file *f = &files[fd];
+	struct open_file *f;
 
-	if ((unsigned)fd >= SOPEN_MAX || f->f_flags == 0) {
+	f = fd2open_file(fd);
+	if (f == NULL || f->f_flags == 0) {
 		errno = EBADF;
 		return (-1);
 	}
diff --git a/stand/libsa/net.h b/stand/libsa/net.h
index 36b7cacc95ef..77a11c8dcab6 100644
--- a/stand/libsa/net.h
+++ b/stand/libsa/net.h
@@ -97,8 +97,6 @@ extern	u_int intf_mtu;
 
 extern	int debug;			/* defined in the machdep sources */
 
-extern struct iodesc sockets[SOPEN_MAX];
-
 /* ARP/RevARP functions: */
 u_char	*arpwhohas(struct iodesc *, struct in_addr);
 void	arp_reply(struct iodesc *, void *);
diff --git a/stand/libsa/netif.c b/stand/libsa/netif.c
index d255cc663d5a..2d32ccd9de7e 100644
--- a/stand/libsa/netif.c
+++ b/stand/libsa/netif.c
@@ -47,7 +47,19 @@ __FBSDID("$FreeBSD$");
 #include "net.h"
 #include "netif.h"
 
-struct iodesc sockets[SOPEN_MAX];
+typedef TAILQ_HEAD(socket_list, iodesc) socket_list_t;
+
+/*
+ * Open socket list. The current implementation and assumption is,
+ * we only remove entries from tail and we only add new entries to tail.
+ * This decision is to keep iodesc id management simple - we get list
+ * entries ordered by continiously growing io_id field.
+ * If we do have multiple sockets open and we do close socket not from tail,
+ * this entry will be marked unused. netif_open() will reuse unused entry, or
+ * netif_close() will free all unused tail entries.
+ */
+static socket_list_t sockets = TAILQ_HEAD_INITIALIZER(sockets);
+
 #ifdef NETIF_DEBUG
 int netif_debug = 0;
 #endif
@@ -63,7 +75,7 @@ netif_init(void)
 {
 	struct netif_driver *drv;
 	int d, i;
-    
+
 #ifdef NETIF_DEBUG
 	if (netif_debug)
 		printf("netif_init: called\n");
@@ -108,7 +120,7 @@ netif_select(void *machdep_hint)
 		for (u = 0; u < drv->netif_nifs; u++) {
 			cur_if.nif_unit = u;
 			unit_done = 0;
-		
+
 #ifdef NETIF_DEBUG
 			if (netif_debug)
 				printf("\t%s%d:", drv->netif_bname,
@@ -179,14 +191,14 @@ netif_attach(struct netif *nif, struct iodesc *desc, void *machdep_hint)
 	if (netif_debug)
 		printf("%s%d: netif_attach\n", drv->netif_bname, nif->nif_unit);
 #endif
-	desc->io_netif = nif; 
+	desc->io_netif = nif;
 #ifdef PARANOID
 	if (drv->netif_init == NULL)
 		panic("%s%d: no netif_init support", drv->netif_bname,
 		    nif->nif_unit);
 #endif
 	drv->netif_init(desc, machdep_hint);
-	bzero(drv->netif_ifs[nif->nif_unit].dif_stats, 
+	bzero(drv->netif_ifs[nif->nif_unit].dif_stats,
 	    sizeof(struct netif_stats));
 }
 
@@ -261,35 +273,71 @@ netif_put(struct iodesc *desc, void *pkt, size_t len)
 	return (rv);
 }
 
+/*
+ * socktodesc_impl:
+ *
+ * Walk socket list and return pointer to iodesc structure.
+ * if id is < 0, return first unused iodesc.
+ */
+static struct iodesc *
+socktodesc_impl(int socket)
+{
+	struct iodesc *s;
+
+	TAILQ_FOREACH(s, &sockets, io_link) {
+		/* search by socket id */
+		if (socket >= 0) {
+			if (s->io_id == socket)
+				break;
+			continue;
+		}
+		/* search for first unused entry */
+		if (s->io_netif == NULL)
+			break;
+	}
+	return (s);
+}
+
 struct iodesc *
 socktodesc(int sock)
 {
-	if (sock >= SOPEN_MAX) {
+	struct iodesc *desc;
+
+	if (sock < 0)
+		desc = NULL;
+	else
+		desc = socktodesc_impl(sock);
+
+	if (desc == NULL)
 		errno = EBADF;
-		return (NULL);
-	}
-	return (&sockets[sock]);
+
+	return (desc);
 }
 
 int
 netif_open(void *machdep_hint)
 {
-	int fd;
 	struct iodesc *s;
 	struct netif *nif;
-	
+
 	/* find a free socket */
-	for (fd = 0, s = sockets; fd < SOPEN_MAX; fd++, s++)
-		if (s->io_netif == (struct netif *)0)
-			goto fnd;
-	errno = EMFILE;
-	return (-1);
-
-fnd:
-	bzero(s, sizeof(*s));
+	s = socktodesc_impl(-1);
+	if (s == NULL) {
+		struct iodesc *last;
+
+		s = calloc(1, sizeof (*s));
+		if (s == NULL)
+			return (-1);
+
+		last = TAILQ_LAST(&sockets, socket_list);
+		if (last != NULL)
+			s->io_id = last->io_id + 1;
+		TAILQ_INSERT_TAIL(&sockets, s, io_link);
+	}
+
 	netif_init();
 	nif = netif_select(machdep_hint);
-	if (!nif) 
+	if (!nif)
 		panic("netboot: no interfaces left untried");
 	if (netif_probe(nif, machdep_hint)) {
 		printf("netboot: couldn't probe %s%d\n",
@@ -299,18 +347,42 @@ fnd:
 	}
 	netif_attach(nif, s, machdep_hint);
 
-	return (fd);
+	return (s->io_id);
 }
 
 int
 netif_close(int sock)
 {
-	if (sock >= SOPEN_MAX) {
-		errno = EBADF;
+	struct iodesc *s, *last;
+	int err;
+
+	err = 0;
+	s = socktodesc_impl(sock);
+	if (s == NULL || sock < 0) {
+		err = EBADF;
+		return (-1);
+	}
+	netif_detach(s->io_netif);
+	bzero(&s->destip, sizeof (s->destip));
+	bzero(&s->myip, sizeof (s->myip));
+	s->destport = 0;
+	s->myport = 0;
+	s->xid = 0;
+	bzero(s->myea, sizeof (s->myea));
+	s->io_netif = NULL;
+
+	/* free unused entries from tail. */
+	TAILQ_FOREACH_REVERSE_SAFE(last, &sockets, socket_list, io_link, s) {
+		if (last->io_netif != NULL)
+			break;
+		TAILQ_REMOVE(&sockets, last, io_link);
+		free(last);
+	}
+
+	if (err) {
+		errno = err;
 		return (-1);
 	}
-	netif_detach(sockets[sock].io_netif);
-	sockets[sock].io_netif = (struct netif *)0;
 
 	return (0);
 }
diff --git a/stand/libsa/open.c b/stand/libsa/open.c
index 9590508b0015..c62e48f0e1d8 100644
--- a/stand/libsa/open.c
+++ b/stand/libsa/open.c
@@ -67,17 +67,66 @@ __FBSDID("$FreeBSD$");
 
 struct fs_ops *exclusive_file_system;
 
-struct open_file files[SOPEN_MAX];
+/*
+ * Open file list. The current implementation and assumption is,
+ * we only remove entries from tail and we only add new entries to tail.
+ * This decision is to keep file id management simple - we get list
+ * entries ordered by continiously growing f_id field.
+ * If we do have multiple files open and we do close file not from tail,
+ * this entry will be marked unused. open() will reuse unused entry, or
+ * close will free all unused tail entries.
+ *
+ * Only case we expect open file list to grow long, is with zfs pools with
+ * many disks. 
+ */
+file_list_t files = TAILQ_HEAD_INITIALIZER(files);
+
+/*
+ * Walk file list and return pointer to open_file structure.
+ * if fd is < 0, return first unused open_file.
+ */
+struct open_file *
+fd2open_file(int fd)
+{
+	struct open_file *f;
+
+	TAILQ_FOREACH(f, &files, f_link) {
+		if (fd >= 0) {
+			if (f->f_id == fd)
+				break;
+			continue;
+		}
+
+		if (f->f_flags == 0)
+			break;
+	}
+	return (f);
+}
 
 static int
-o_gethandle(void)
+o_gethandle(struct open_file **ptr)
 {
-	int fd;
+	struct open_file *f, *last;
 
-	for (fd = 0; fd < SOPEN_MAX; fd++)
-		if (files[fd].f_flags == 0)
-			return (fd);
-	return (-1);
+	/* Pick up unused entry */
+	f = fd2open_file(-1);
+	if (f != NULL) {
+		*ptr = f;
+		return (f->f_id);
+	}
+
+	/* Add new entry */
+	f = calloc(1, sizeof (*f));
+	if (f == NULL)
+		return (-1);
+
+	last = TAILQ_LAST(&files, file_list);
+	if (last != NULL)
+		f->f_id = last->f_id + 1;
+	TAILQ_INSERT_TAIL(&files, f, f_link);
+
+	*ptr = f;
+	return (f->f_id);
 }
 
 static void
@@ -98,12 +147,11 @@ open(const char *fname, int mode)
 
 	TSENTER();
 
-	if ((fd = o_gethandle()) == -1) {
+	if ((fd = o_gethandle(&f)) == -1) {
 		errno = EMFILE;
 		return (-1);
 	}
 
-	f = &files[fd];
 	f->f_flags = mode + 1;
 	f->f_dev = NULL;
 	f->f_ops = NULL;
diff --git a/stand/libsa/read.c b/stand/libsa/read.c
index 7ad75c387a5d..d079944bf5d0 100644
--- a/stand/libsa/read.c
+++ b/stand/libsa/read.c
@@ -69,12 +69,13 @@ __FBSDID("$FreeBSD$");
 ssize_t
 read(int fd, void *dest, size_t bcount)
 {
-	struct open_file *f = &files[fd];
+	struct open_file *f;
 	size_t resid;
 
 	TSENTER();
 
-	if ((unsigned)fd >= SOPEN_MAX || !(f->f_flags & F_READ)) {
+	f = fd2open_file(fd);
+	if (f == NULL || !(f->f_flags & F_READ)) {
 		errno = EBADF;
 		return (-1);
 	}
diff --git a/stand/libsa/readdir.c b/stand/libsa/readdir.c
index e49d93d15ed6..7757647e11e8 100644
--- a/stand/libsa/readdir.c
+++ b/stand/libsa/readdir.c
@@ -34,9 +34,10 @@ struct dirent *
 readdirfd(int fd)
 {
 	static struct dirent dir;		/* XXX not thread safe */
-	struct open_file *f = &files[fd];
+	struct open_file *f;
 
-	if ((unsigned)fd >= SOPEN_MAX || !(f->f_flags & F_READ)) {
+	f = fd2open_file(fd);
+	if (f == NULL || !(f->f_flags & F_READ)) {
 		errno = EBADF;
 		return (NULL);
 	}
diff --git a/stand/libsa/stand.h b/stand/libsa/stand.h
index eb95afe4b169..535fee31d586 100644
--- a/stand/libsa/stand.h
+++ b/stand/libsa/stand.h
@@ -65,6 +65,7 @@
 #include <sys/cdefs.h>
 #include <sys/stat.h>
 #include <sys/dirent.h>
+#include <sys/queue.h>
 
 /* this header intentionally exports NULL from <string.h> */
 #include <string.h>
@@ -182,11 +183,14 @@ struct open_file {
     char		*f_rabuf;	/* readahead buffer pointer */
     size_t		f_ralen;	/* valid data in readahead buffer */
     off_t		f_raoffset;	/* consumer offset in readahead buffer */
+    int			f_id;		/* file number */
+    TAILQ_ENTRY(open_file) f_link;	/* next entry */
 #define SOPEN_RASIZE	512
 };
 
-#define	SOPEN_MAX	64
-extern struct open_file files[];
+typedef TAILQ_HEAD(file_list, open_file) file_list_t;
+extern file_list_t files;
+extern struct open_file *fd2open_file(int);
 
 /* f_flags values */
 #define	F_READ		0x0001	/* file opened for reading */
diff --git a/stand/libsa/write.c b/stand/libsa/write.c
index 7d054e005cda..db82d7eaf4f6 100644
--- a/stand/libsa/write.c
+++ b/stand/libsa/write.c
@@ -69,10 +69,11 @@ __FBSDID("$FreeBSD$");
 ssize_t
 write(int fd, const void *dest, size_t bcount)
 {
-	struct open_file *f = &files[fd];
+	struct open_file *f;
 	size_t resid;
 
-	if ((unsigned)fd >= SOPEN_MAX || !(f->f_flags & F_WRITE)) {
+	f = fd2open_file(fd);
+	if (f == NULL || !(f->f_flags & F_WRITE)) {
 		errno = EBADF;
 		return (-1);
 	}