rtld: stop using unbound alloca()

For DoneList allocations, its size depends on the number of loaded DSOs.
Small images could be served by alloca(), but large donelists need to
go into heap.

For map_object(), alloca size is the number of segments in the object.

In both cases, over-grown situations would cause a stack overflow.

PR:	295991
Noted and reviewed by:	kevans
Tested by:	Marek Zarychta <zarychtam@plan-b.pwste.edu.pl>
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D57908

3 files changed, 63 insertions, 25 deletions

diff --git a/libexec/rtld-elf/map_object.c b/libexec/rtld-elf/map_object.c
index 6c6f0c9289b9..d769d01d3b24 100644
--- a/libexec/rtld-elf/map_object.c
+++ b/libexec/rtld-elf/map_object.c
@@ -113,7 +113,7 @@ map_object(int fd, const char *path, const struct stat *sb, bool ismain)
 	note_end = 0;
 	note_map = NULL;
 	note_map_len = 0;
-	segs = alloca(sizeof(segs[0]) * hdr->e_phnum);
+	segs = xcalloc(hdr->e_phnum, sizeof(segs[0]));
 	stack_flags = PF_X | PF_R | PF_W;
 	text_end = 0;
 	while (phdr < phlimit) {
@@ -341,6 +341,7 @@ map_object(int fd, const char *path, const struct stat *sb, bool ismain)
 	if (note_map != NULL)
 		munmap(note_map, note_map_len);
 	munmap(hdr, page_size);
+	free(segs);
 	return (obj);
 
 error1:
@@ -351,6 +352,7 @@ error:
 	if (!phdr_in_zero_page(hdr))
 		munmap(phdr, hdr->e_phnum * sizeof(phdr[0]));
 	munmap(hdr, page_size);
+	free(segs);
 	return (NULL);
 }
 
diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index 938e4b0b8a6b..864148df99ba 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -342,16 +342,6 @@ const char *ld_env_prefix = LD_;
 
 static void (*rtld_exit_ptr)(void);
 
-/*
- * Fill in a DoneList with an allocation large enough to hold all of
- * the currently-loaded objects.  Keep this as a macro since it calls
- * alloca and we want that to occur within the scope of the caller.
- */
-#define donelist_init(dlp)                                             \
-	((dlp)->objs = alloca(obj_count * sizeof(dlp)->objs[0]),       \
-	    assert((dlp)->objs != NULL), (dlp)->num_alloc = obj_count, \
-	    (dlp)->num_used = 0)
-
 #define LD_UTRACE(e, h, mb, ms, r, n)                      \
 	do {                                               \
 		if (ld_utrace != NULL)                     \
@@ -1042,6 +1032,38 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 	return ((func_ptr_type)obj_main->entry);
 }
 
+/*
+ * Fill in a DoneList with an allocation large enough to hold all of
+ * the currently-loaded   Keep this as a macro since it calls
+ * alloca and we want that to occur within the scope of the caller.
+ */
+#define	DLP_ALLOCA_LIMIT	100	/* 800 bytes on LP64 */
+#define donelist_init(_DLP, _REQ)	do {				\
+	DoneList *_dlp = _DLP;						\
+	SymLook *_r = _REQ;						\
+	_dlp->num_alloc = obj_count,					\
+	_dlp->req = NULL;						\
+	if (_dlp->num_alloc > DLP_ALLOCA_LIMIT) {			\
+		_dlp->objs = xcalloc(_dlp->num_alloc, sizeof(_dlp->objs[0])); \
+		if (_r != NULL && _r->donelist_mem == NULL) {		\
+			_r->donelist_mem = _dlp->objs;			\
+			_dlp->req = _r;				\
+		}							\
+	} else {							\
+		_dlp->objs = alloca(_dlp->num_alloc * sizeof(_dlp->objs[0])); \
+	}								\
+	_dlp->num_used = 0;						\
+} while (0)
+
+static void
+donelist_free(DoneList *dlp)
+{
+	if (dlp->num_alloc > DLP_ALLOCA_LIMIT)
+		free(dlp->objs);
+	if (dlp->req != NULL)
+		dlp->req->donelist_mem = NULL;
+}
+
 void *
 rtld_resolve_ifunc(const Obj_Entry *obj, const Elf_Sym *def)
 {
@@ -2319,7 +2341,7 @@ init_dag(Obj_Entry *root)
 
 	if (root->dag_inited)
 		return;
-	donelist_init(&donelist);
+	donelist_init(&donelist, NULL);
 
 	/* Root object belongs to own DAG. */
 	objlist_push_tail(&root->dldags, root);
@@ -2342,6 +2364,7 @@ init_dag(Obj_Entry *root)
 		}
 	}
 	root->dag_inited = true;
+	donelist_free(&donelist);
 }
 
 static void
@@ -4103,8 +4126,11 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
 
 	LD_UTRACE(UTRACE_DLSYM_START, handle, NULL, 0, 0, name);
 	rlock_acquire(rtld_bind_lock, &lockstate);
-	if (sigsetjmp(lockstate.env, 0) != 0)
+	if (sigsetjmp(lockstate.env, 0) != 0) {
 		lock_upgrade(rtld_bind_lock, &lockstate);
+		free(req.donelist_mem);
+		req.donelist_mem = NULL;
+	}
 	if (handle == NULL || handle == RTLD_NEXT || handle == RTLD_DEFAULT ||
 	    handle == RTLD_SELF) {
 		if ((obj = obj_from_addr(retaddr)) == NULL) {
@@ -4173,7 +4199,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
 			return (NULL);
 		}
 
-		donelist_init(&donelist);
+		donelist_init(&donelist, &req);
 		if (obj->mainprog) {
 			/* Handle obtained by dlopen(NULL, ...) implies global
 			 * scope. */
@@ -4204,6 +4230,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
 				defobj = req.defobj_out;
 			}
 		}
+		donelist_free(&donelist);
 	}
 
 	if (def != NULL) {
@@ -4731,21 +4758,24 @@ get_program_var_addr(const char *name, RtldLockState *lockstate)
 {
 	SymLook req;
 	DoneList donelist;
+	const void **res;
 
 	symlook_init(&req, name);
 	req.lockstate = lockstate;
-	donelist_init(&donelist);
+	donelist_init(&donelist, NULL);
 	if (symlook_global(&req, &donelist) != 0)
 		return (NULL);
 	if (ELF_ST_TYPE(req.sym_out->st_info) == STT_FUNC)
-		return ((const void **)make_function_pointer(req.sym_out,
-		    req.defobj_out));
+		res = (const void **)make_function_pointer(req.sym_out,
+		    req.defobj_out);
 	else if (ELF_ST_TYPE(req.sym_out->st_info) == STT_GNU_IFUNC)
-		return ((const void **)rtld_resolve_ifunc(req.defobj_out,
-		    req.sym_out));
+		res = (const void **)rtld_resolve_ifunc(req.defobj_out,
+		    req.sym_out);
 	else
-		return ((const void **)(req.defobj_out->relocbase +
-		    req.sym_out->st_value));
+		res = (const void **)(req.defobj_out->relocbase +
+		    req.sym_out->st_value);
+	donelist_free(&donelist);
+	return (res);
 }
 
 /*
@@ -4820,7 +4850,7 @@ symlook_default(SymLook *req, const Obj_Entry *refobj)
 	SymLook req1;
 	int res;
 
-	donelist_init(&donelist);
+	donelist_init(&donelist, req);
 	symlook_init_from_req(&req1, req);
 
 	/*
@@ -4873,6 +4903,7 @@ symlook_default(SymLook *req, const Obj_Entry *refobj)
 		}
 	}
 
+	donelist_free(&donelist);
 	return (req->sym_out != NULL ? 0 : ESRCH);
 }
 
@@ -4954,13 +4985,15 @@ symlook_obj_load_filtees(SymLook *req, SymLook *req1, const Obj_Entry *obj,
     Needed_Entry *needed)
 {
 	DoneList donelist;
-	int flags;
+	int flags, res;
 
 	flags = (req->flags & SYMLOOK_EARLY) != 0 ? RTLD_LO_EARLY : 0;
 	load_filtees(__DECONST(Obj_Entry *, obj), flags, req->lockstate);
-	donelist_init(&donelist);
+	donelist_init(&donelist, NULL);
 	symlook_init_from_req(req1, req);
-	return (symlook_needed(req1, needed, &donelist));
+	res = symlook_needed(req1, needed, &donelist);
+	donelist_free(&donelist);
+	return (res);
 }
 
 /*
@@ -6305,6 +6338,7 @@ symlook_init_from_req(SymLook *dst, const SymLook *src)
 	dst->defobj_out = NULL;
 	dst->sym_out = NULL;
 	dst->lockstate = src->lockstate;
+	dst->donelist_mem = NULL;
 }
 
 static int
diff --git a/libexec/rtld-elf/rtld.h b/libexec/rtld-elf/rtld.h
index 322d1ee94d6f..bdcfc3dc5ded 100644
--- a/libexec/rtld-elf/rtld.h
+++ b/libexec/rtld-elf/rtld.h
@@ -319,6 +319,7 @@ typedef struct Struct_SymCache {
  */
 typedef struct Struct_DoneList {
     const Obj_Entry **objs;		/* Array of object pointers */
+    struct Struct_SymLook *req;
     unsigned int num_alloc;		/* Allocated size of the array */
     unsigned int num_used;		/* Number of array slots used */
 } DoneList;
@@ -348,6 +349,7 @@ typedef struct Struct_SymLook {
     const Obj_Entry *defobj_out;
     const Elf_Sym *sym_out;
     struct Struct_RtldLockState *lockstate;
+    void *donelist_mem;
 } SymLook;
 
 enum {