heimdal: Vendor import f62e2f278vendor/heimdal/7.8.0-2023-06-10-f62e2f278vendor/heimdal

Heimdal 7.8.0 does not support OpenSSL 3.0. 7.9.0 will but it hasn't
been released yet. We are importing f62e2f278 for its OpenSSL 3.0
support.

1 files changed, 317 insertions, 320 deletions

diff --git a/lib/hcrypto/rsa-ltm.c b/lib/hcrypto/rsa-ltm.c
index 4ac5617cb6ba..49e35f5a38a9 100644
--- a/lib/hcrypto/rsa-ltm.c
+++ b/lib/hcrypto/rsa-ltm.c
@@ -40,96 +40,122 @@
 
 #include "tommath.h"
 
-static int
+#define CHECK(f)                                                        \
+    do { where = __LINE__ + 1; if (ret == MP_OKAY && ((ret = f)) != MP_OKAY) { goto out; } } while (0)
+#define FIRST(e) do { ret = (e); } while (0)
+#define FIRST_ALLOC(e)                                                  \
+    do { where = __LINE__; ret = ((e)) ? MP_OKAY : MP_MEM; } while (0)
+#define THEN_MP(e)                                                      \
+    do { where = __LINE__ + 1; if (ret == MP_OKAY) ret = (e); } while (0)
+#define THEN_IF_MP(cond, e)                                             \
+    do { where = __LINE__ + 1; if (ret == MP_OKAY && (cond)) ret = (e); } while (0)
+#define THEN_IF_VOID(cond, e)                                           \
+    do { where = __LINE__ + 1; if (ret == MP_OKAY && (cond)) e; } while (0)
+#define THEN_VOID(e)                                                    \
+    do { where = __LINE__ + 1; if (ret == MP_OKAY) e; } while (0)
+#define THEN_ALLOC(e)                                                   \
+    do { where = __LINE__ + 1; if (ret == MP_OKAY) ret = ((e)) ? MP_OKAY : MP_MEM; } while (0)
+
+static mp_err
 random_num(mp_int *num, size_t len)
 {
     unsigned char *p;
+    mp_err ret = MP_MEM;
 
-    len = (len + 7) / 8;
-    p = malloc(len);
-    if (p == NULL)
-	return 1;
-    if (RAND_bytes(p, len) != 1) {
-	free(p);
-	return 1;
-    }
-    mp_read_unsigned_bin(num, p, len);
+    len = (len + 7) / 8; /* bits to bytes */
+    if ((p = malloc(len)) && RAND_bytes(p, len) != 1)
+        ret = MP_ERR;
+    if (p)
+        ret = mp_from_ubin(num, p, len);
     free(p);
-    return 0;
+    return ret;
 }
 
-static void
+static mp_err
 BN2mpz(mp_int *s, const BIGNUM *bn)
 {
     size_t len;
+    mp_err ret = MP_MEM;
     void *p;
 
     len = BN_num_bytes(bn);
     p = malloc(len);
-    BN_bn2bin(bn, p);
-    mp_read_unsigned_bin(s, p, len);
+    if (p) {
+        BN_bn2bin(bn, p);
+        ret = mp_from_ubin(s, p, len);
+    }
     free(p);
+    return ret;
 }
 
-static void
+static mp_err
 setup_blind(mp_int *n, mp_int *b, mp_int *bi)
 {
-    random_num(b, mp_count_bits(n));
-    mp_mod(b, n, b);
-    mp_invmod(b, n, bi);
+    mp_err ret;
+
+    ret = random_num(b, mp_count_bits(n));
+    if (ret == MP_OKAY) ret = mp_mod(b, n, b);
+    if (ret == MP_OKAY) ret = mp_invmod(b, n, bi);
+    return ret;
 }
 
-static void
+static mp_err
 blind(mp_int *in, mp_int *b, mp_int *e, mp_int *n)
 {
+    mp_err ret;
     mp_int t1;
-    mp_init(&t1);
+
+    ret = mp_init(&t1);
     /* in' = (in * b^e) mod n */
-    mp_exptmod(b, e, n, &t1);
-    mp_mul(&t1, in, in);
-    mp_mod(in, n, in);
+    if (ret == MP_OKAY) ret = mp_exptmod(b, e, n, &t1);
+    if (ret == MP_OKAY) ret = mp_mul(&t1, in, in);
+    if (ret == MP_OKAY) ret = mp_mod(in, n, in);
     mp_clear(&t1);
+    return ret;
 }
 
-static void
+static mp_err
 unblind(mp_int *out, mp_int *bi, mp_int *n)
 {
+    mp_err ret;
+
     /* out' = (out * 1/b) mod n */
-    mp_mul(out, bi, out);
-    mp_mod(out, n, out);
+    ret = mp_mul(out, bi, out);
+    if (ret == MP_OKAY) ret = mp_mod(out, n, out);
+    return ret;
 }
 
-static int
+static mp_err
 ltm_rsa_private_calculate(mp_int * in, mp_int * p,  mp_int * q,
 			  mp_int * dmp1, mp_int * dmq1, mp_int * iqmp,
 			  mp_int * out)
 {
+    mp_err ret;
     mp_int vp, vq, u;
+    int where HEIMDAL_UNUSED_ATTRIBUTE = 0;
 
-    mp_init_multi(&vp, &vq, &u, NULL);
+    FIRST(mp_init_multi(&vp, &vq, &u, NULL));
 
     /* vq = c ^ (d mod (q - 1)) mod q */
     /* vp = c ^ (d mod (p - 1)) mod p */
-    mp_mod(in, p, &u);
-    mp_exptmod(&u, dmp1, p, &vp);
-    mp_mod(in, q, &u);
-    mp_exptmod(&u, dmq1, q, &vq);
+    THEN_MP(mp_mod(in, p, &u));
+    THEN_MP(mp_exptmod(&u, dmp1, p, &vp));
+    THEN_MP(mp_mod(in, q, &u));
+    THEN_MP(mp_exptmod(&u, dmq1, q, &vq));
 
     /* C2 = 1/q mod p  (iqmp) */
     /* u = (vp - vq)C2 mod p. */
-    mp_sub(&vp, &vq, &u);
-    if (mp_isneg(&u))
-	mp_add(&u, p, &u);
-    mp_mul(&u, iqmp, &u);
-    mp_mod(&u, p, &u);
+    THEN_MP(mp_sub(&vp, &vq, &u));
+    THEN_IF_MP(mp_isneg(&u), mp_add(&u, p, &u));
+    THEN_MP(mp_mul(&u, iqmp, &u));
+    THEN_MP(mp_mod(&u, p, &u));
 
     /* c ^ d mod n = vq + u q */
-    mp_mul(&u, q, &u);
-    mp_add(&u, &vq, out);
+    THEN_MP(mp_mul(&u, q, &u));
+    THEN_MP(mp_add(&u, &vq, out));
 
     mp_clear_multi(&vp, &vq, &u, NULL);
-
-    return 0;
+    return ret;
 }
 
 /*
@@ -140,78 +166,56 @@ static int
 ltm_rsa_public_encrypt(int flen, const unsigned char* from,
 			unsigned char* to, RSA* rsa, int padding)
 {
-    unsigned char *p, *p0;
-    int res;
-    size_t size, padlen;
+    unsigned char *p = NULL, *p0 = NULL;
+    size_t size, ssize, padlen;
     mp_int enc, dec, n, e;
+    mp_err ret;
+    int where = __LINE__;
 
     if (padding != RSA_PKCS1_PADDING)
 	return -1;
 
-    mp_init_multi(&n, &e, &enc, &dec, NULL);
+    FIRST(mp_init_multi(&n, &e, &enc, &dec, NULL));
 
     size = RSA_size(rsa);
-
-    if (size < RSA_PKCS1_PADDING_SIZE || size - RSA_PKCS1_PADDING_SIZE < flen) {
-	mp_clear_multi(&n, &e, &enc, &dec, NULL);
-	return -2;
-    }
-
-    BN2mpz(&n, rsa->n);
-    BN2mpz(&e, rsa->e);
-
-    if (mp_cmp_d(&e, 3) == MP_LT) {
-	mp_clear_multi(&e, &n, &enc, &dec, NULL);
-	return -2;
-    }
-
-    p = p0 = malloc(size - 1);
-    if (p0 == NULL) {
-	mp_clear_multi(&e, &n, &enc, &dec, NULL);
-	return -3;
-    }
-
-    padlen = size - flen - 3;
-
-    *p++ = 2;
-    if (RAND_bytes(p, padlen) != 1) {
-	mp_clear_multi(&e, &n, &enc, &dec, NULL);
-	free(p0);
-	return -4;
+    THEN_IF_MP((size < RSA_PKCS1_PADDING_SIZE ||
+                size - RSA_PKCS1_PADDING_SIZE < flen),
+               MP_ERR);
+    THEN_MP(BN2mpz(&n, rsa->n));
+    THEN_MP(BN2mpz(&e, rsa->e));
+    THEN_IF_MP((mp_cmp_d(&e, 3) == MP_LT), MP_ERR);
+    THEN_ALLOC((p = p0 = malloc(size - 1)));
+
+    if (ret == MP_OKAY) {
+        padlen = size - flen - 3;
+        *p++ = 2;
     }
-    while(padlen) {
-	if (*p == 0)
-	    *p = 1;
-	padlen--;
-	p++;
+    THEN_IF_MP((RAND_bytes(p, padlen) != 1), MP_ERR);
+
+    if (ret == MP_OKAY) {
+        while (padlen) {
+            if (*p == 0)
+                *p = 1;
+            padlen--;
+            p++;
+        }
+        *p++ = 0;
+        memcpy(p, from, flen);
+        p += flen;
+        assert((p - p0) == size - 1);
     }
-    *p++ = 0;
-    memcpy(p, from, flen);
-    p += flen;
-    assert((p - p0) == size - 1);
 
-    mp_read_unsigned_bin(&dec, p0, size - 1);
-    free(p0);
-
-    res = mp_exptmod(&dec, &e, &n, &enc);
+    THEN_MP(mp_from_ubin(&dec, p0, size - 1));
+    THEN_MP(mp_exptmod(&dec, &e, &n, &enc));
+    THEN_VOID(ssize = mp_ubin_size(&enc));
+    THEN_VOID(assert(size >= ssize));
+    THEN_MP(mp_to_ubin(&enc, to, SIZE_MAX, NULL));
+    THEN_VOID(size = ssize);
 
     mp_clear_multi(&dec, &e, &n, NULL);
-
-    if (res != 0) {
-	mp_clear(&enc);
-	return -4;
-    }
-
-    {
-	size_t ssize;
-	ssize = mp_unsigned_bin_size(&enc);
-	assert(size >= ssize);
-	mp_to_unsigned_bin(&enc, to);
-	size = ssize;
-    }
     mp_clear(&enc);
-
-    return size;
+    free(p0);
+    return ret == MP_OKAY ? size : -where;
 }
 
 static int
@@ -219,9 +223,10 @@ ltm_rsa_public_decrypt(int flen, const unsigned char* from,
 		       unsigned char* to, RSA* rsa, int padding)
 {
     unsigned char *p;
-    int res;
+    mp_err ret;
     size_t size;
     mp_int s, us, n, e;
+    int where = __LINE__;
 
     if (padding != RSA_PKCS1_PADDING)
 	return -1;
@@ -229,55 +234,38 @@ ltm_rsa_public_decrypt(int flen, const unsigned char* from,
     if (flen > RSA_size(rsa))
 	return -2;
 
-    mp_init_multi(&e, &n, &s, &us, NULL);
-
-    BN2mpz(&n, rsa->n);
-    BN2mpz(&e, rsa->e);
+    FIRST(mp_init_multi(&e, &n, &s, &us, NULL));
+    THEN_MP(BN2mpz(&n, rsa->n));
+    THEN_MP(BN2mpz(&e, rsa->e));
+    THEN_MP((mp_cmp_d(&e, 3) == MP_LT) ? MP_ERR : MP_OKAY);
+    THEN_MP(mp_from_ubin(&s, rk_UNCONST(from), (size_t)flen));
+    THEN_MP((mp_cmp(&s, &n) >= 0) ? MP_ERR : MP_OKAY);
+    THEN_MP(mp_exptmod(&s, &e, &n, &us));
 
-    if (mp_cmp_d(&e, 3) == MP_LT) {
-	mp_clear_multi(&e, &n, &s, &us, NULL);
-	return -3;
-    }
-
-    mp_read_unsigned_bin(&s, rk_UNCONST(from), flen);
-
-    if (mp_cmp(&s, &n) >= 0) {
-	mp_clear_multi(&e, &n, &s, &us, NULL);
-	return -4;
-    }
-
-    res = mp_exptmod(&s, &e, &n, &us);
+    THEN_VOID(p = to);
+    THEN_VOID(size = mp_ubin_size(&us));
+    THEN_VOID(assert(size <= RSA_size(rsa)));
+    THEN_MP(mp_to_ubin(&us, p, SIZE_MAX, NULL));
 
     mp_clear_multi(&e, &n, &s, NULL);
-
-    if (res != 0) {
-	mp_clear(&us);
-	return -5;
-    }
-    p = to;
-
-
-    size = mp_unsigned_bin_size(&us);
-    assert(size <= RSA_size(rsa));
-    mp_to_unsigned_bin(&us, p);
-
     mp_clear(&us);
 
+    if (ret != MP_OKAY || size == 0)
+        return -where;
+
     /* head zero was skipped by mp_to_unsigned_bin */
     if (*p == 0)
-	return -6;
+        return -where;
     if (*p != 1)
-	return -7;
+        return -(where + 1);
     size--; p++;
     while (size && *p == 0xff) {
-	size--; p++;
+        size--; p++;
     }
     if (size == 0 || *p != 0)
-	return -8;
+        return -(where + 2);
     size--; p++;
-
     memmove(to, p, size);
-
     return size;
 }
 
@@ -285,102 +273,88 @@ static int
 ltm_rsa_private_encrypt(int flen, const unsigned char* from,
 			unsigned char* to, RSA* rsa, int padding)
 {
-    unsigned char *ptr, *ptr0;
-    int res;
-    int size;
+    unsigned char *ptr, *ptr0 = NULL;
+    mp_err ret;
     mp_int in, out, n, e;
     mp_int bi, b;
+    size_t size;
     int blinding = (rsa->flags & RSA_FLAG_NO_BLINDING) == 0;
     int do_unblind = 0;
+    int where = __LINE__;
 
     if (padding != RSA_PKCS1_PADDING)
 	return -1;
 
-    mp_init_multi(&e, &n, &in, &out, &b, &bi, NULL);
+    FIRST(mp_init_multi(&e, &n, &in, &out, &b, &bi, NULL));
 
     size = RSA_size(rsa);
-
     if (size < RSA_PKCS1_PADDING_SIZE || size - RSA_PKCS1_PADDING_SIZE < flen)
 	return -2;
 
-    ptr0 = ptr = malloc(size);
-    *ptr++ = 0;
-    *ptr++ = 1;
-    memset(ptr, 0xff, size - flen - 3);
-    ptr += size - flen - 3;
-    *ptr++ = 0;
-    memcpy(ptr, from, flen);
-    ptr += flen;
-    assert((ptr - ptr0) == size);
-
-    BN2mpz(&n, rsa->n);
-    BN2mpz(&e, rsa->e);
-
-    if (mp_cmp_d(&e, 3) == MP_LT) {
-	size = -3;
-	goto out;
+    THEN_ALLOC((ptr0 = ptr = malloc(size)));
+    if (ret == MP_OKAY) {
+        *ptr++ = 0;
+        *ptr++ = 1;
+        memset(ptr, 0xff, size - flen - 3);
+        ptr += size - flen - 3;
+        *ptr++ = 0;
+        memcpy(ptr, from, flen);
+        ptr += flen;
+        assert((ptr - ptr0) == size);
     }
 
-    mp_read_unsigned_bin(&in, ptr0, size);
+    THEN_MP(BN2mpz(&n, rsa->n));
+    THEN_MP(BN2mpz(&e, rsa->e));
+    THEN_IF_MP((mp_cmp_d(&e, 3) == MP_LT), MP_ERR);
+    THEN_MP(mp_from_ubin(&in, ptr0, size));
     free(ptr0);
 
-    if(mp_isneg(&in) || mp_cmp(&in, &n) >= 0) {
-	size = -3;
-	goto out;
-    }
+    THEN_IF_MP((mp_isneg(&in) || mp_cmp(&in, &n) >= 0), MP_ERR);
 
     if (blinding) {
-	setup_blind(&n, &b, &bi);
-	blind(&in, &b, &e, &n);
+	THEN_MP(setup_blind(&n, &b, &bi));
+	THEN_MP(blind(&in, &b, &e, &n));
 	do_unblind = 1;
     }
 
-    if (rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 && rsa->iqmp) {
+    if (ret == MP_OKAY && rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 &&
+        rsa->iqmp) {
 	mp_int p, q, dmp1, dmq1, iqmp;
 
-	mp_init_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
-
-	BN2mpz(&p, rsa->p);
-	BN2mpz(&q, rsa->q);
-	BN2mpz(&dmp1, rsa->dmp1);
-	BN2mpz(&dmq1, rsa->dmq1);
-	BN2mpz(&iqmp, rsa->iqmp);
-
-	res = ltm_rsa_private_calculate(&in, &p, &q, &dmp1, &dmq1, &iqmp, &out);
-
+	FIRST(mp_init_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL));
+	THEN_MP(BN2mpz(&p, rsa->p));
+	THEN_MP(BN2mpz(&q, rsa->q));
+	THEN_MP(BN2mpz(&dmp1, rsa->dmp1));
+	THEN_MP(BN2mpz(&dmq1, rsa->dmq1));
+	THEN_MP(BN2mpz(&iqmp, rsa->iqmp));
+        THEN_MP(ltm_rsa_private_calculate(&in, &p, &q, &dmp1, &dmq1, &iqmp,
+                                          &out));
 	mp_clear_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
-
-	if (res != 0) {
-	    size = -4;
-	    goto out;
-	}
-    } else {
+	if (ret != MP_OKAY) goto out;
+    } else if (ret == MP_OKAY) {
 	mp_int d;
 
-	BN2mpz(&d, rsa->d);
-	res = mp_exptmod(&in, &d, &n, &out);
+	THEN_MP(BN2mpz(&d, rsa->d));
+	THEN_MP(mp_exptmod(&in, &d, &n, &out));
 	mp_clear(&d);
-	if (res != 0) {
-	    size = -5;
-	    goto out;
-	}
+	if (ret != MP_OKAY) goto out;
     }
 
     if (do_unblind)
-	unblind(&out, &bi, &n);
+	THEN_MP(unblind(&out, &bi, &n));
 
-    if (size > 0) {
+    if (ret == MP_OKAY && size > 0) {
 	size_t ssize;
-	ssize = mp_unsigned_bin_size(&out);
+
+	ssize = mp_ubin_size(&out);
 	assert(size >= ssize);
-	mp_to_unsigned_bin(&out, to);
+	THEN_MP(mp_to_ubin(&out, to, SIZE_MAX, NULL));
 	size = ssize;
     }
 
  out:
     mp_clear_multi(&e, &n, &in, &out, &b, &bi, NULL);
-
-    return size;
+    return ret == MP_OKAY ? size : -where;
 }
 
 static int
@@ -388,10 +362,12 @@ ltm_rsa_private_decrypt(int flen, const unsigned char* from,
 			unsigned char* to, RSA* rsa, int padding)
 {
     unsigned char *ptr;
-    int res, size;
+    size_t size;
+    mp_err ret;
     mp_int in, out, n, e, b, bi;
     int blinding = (rsa->flags & RSA_FLAG_NO_BLINDING) == 0;
     int do_unblind = 0;
+    int where = __LINE__;
 
     if (padding != RSA_PKCS1_PADDING)
 	return -1;
@@ -400,95 +376,75 @@ ltm_rsa_private_decrypt(int flen, const unsigned char* from,
     if (flen > size)
 	return -2;
 
-    mp_init_multi(&in, &n, &e, &out, &b, &bi, NULL);
-
-    BN2mpz(&n, rsa->n);
-    BN2mpz(&e, rsa->e);
-
-    if (mp_cmp_d(&e, 3) == MP_LT) {
-	size = -2;
-	goto out;
-    }
-
-    mp_read_unsigned_bin(&in, rk_UNCONST(from), flen);
-
-    if(mp_isneg(&in) || mp_cmp(&in, &n) >= 0) {
-	size = -2;
-	goto out;
-    }
+    FIRST(mp_init_multi(&in, &n, &e, &out, &b, &bi, NULL));
+    THEN_MP(BN2mpz(&n, rsa->n));
+    THEN_MP(BN2mpz(&e, rsa->e));
+    THEN_IF_MP((mp_cmp_d(&e, 3) == MP_LT), MP_ERR);
+    THEN_MP(mp_from_ubin(&in, rk_UNCONST(from), flen));
+    THEN_IF_MP((mp_isneg(&in) || mp_cmp(&in, &n) >= 0), MP_ERR);
 
     if (blinding) {
-	setup_blind(&n, &b, &bi);
-	blind(&in, &b, &e, &n);
+	THEN_MP(setup_blind(&n, &b, &bi));
+	THEN_MP(blind(&in, &b, &e, &n));
 	do_unblind = 1;
     }
 
-    if (rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 && rsa->iqmp) {
+    if (ret == MP_OKAY && rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 &&
+        rsa->iqmp) {
 	mp_int p, q, dmp1, dmq1, iqmp;
 
-	mp_init_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
-
-	BN2mpz(&p, rsa->p);
-	BN2mpz(&q, rsa->q);
-	BN2mpz(&dmp1, rsa->dmp1);
-	BN2mpz(&dmq1, rsa->dmq1);
-	BN2mpz(&iqmp, rsa->iqmp);
-
-	res = ltm_rsa_private_calculate(&in, &p, &q, &dmp1, &dmq1, &iqmp, &out);
-
+	THEN_MP(mp_init_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL));
+	THEN_MP(BN2mpz(&p, rsa->p));
+	THEN_MP(BN2mpz(&q, rsa->q));
+	THEN_MP(BN2mpz(&dmp1, rsa->dmp1));
+	THEN_MP(BN2mpz(&dmq1, rsa->dmq1));
+	THEN_MP(BN2mpz(&iqmp, rsa->iqmp));
+	THEN_MP(ltm_rsa_private_calculate(&in, &p, &q, &dmp1, &dmq1, &iqmp, &out));
 	mp_clear_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
-
-	if (res != 0) {
-	    size = -3;
-	    goto out;
-	}
-
-    } else {
+	if (ret != MP_OKAY) goto out;
+    } else if (ret == MP_OKAY) {
 	mp_int d;
 
-	if(mp_isneg(&in) || mp_cmp(&in, &n) >= 0)
-	    return -4;
-
-	BN2mpz(&d, rsa->d);
-	res = mp_exptmod(&in, &d, &n, &out);
+	THEN_IF_MP((mp_isneg(&in) || mp_cmp(&in, &n) >= 0), MP_ERR);
+	THEN_MP(BN2mpz(&d, rsa->d));
+	THEN_MP(mp_exptmod(&in, &d, &n, &out));
 	mp_clear(&d);
-	if (res != 0) {
-	    size = -5;
-	    goto out;
-	}
+	if (ret != MP_OKAY) goto out;
     }
 
     if (do_unblind)
-	unblind(&out, &bi, &n);
+	THEN_MP(unblind(&out, &bi, &n));
 
-    ptr = to;
-    {
-	size_t ssize;
-	ssize = mp_unsigned_bin_size(&out);
-	assert(size >= ssize);
-	mp_to_unsigned_bin(&out, ptr);
+    if (ret == MP_OKAY) {
+        size_t ssize;
+
+        ptr = to;
+        ssize = mp_ubin_size(&out);
+        assert(size >= ssize);
+        ret = mp_to_ubin(&out, ptr, SIZE_MAX, NULL);
+        if (ret != MP_OKAY) goto out;
 	size = ssize;
-    }
 
-    /* head zero was skipped by mp_int_to_unsigned */
-    if (*ptr != 2) {
-	size = -6;
-	goto out;
-    }
-    size--; ptr++;
-    while (size && *ptr != 0) {
-	size--; ptr++;
+        /* head zero was skipped by mp_int_to_unsigned */
+        if (*ptr != 2) {
+            where = __LINE__;
+            goto out;
+        }
+        size--; ptr++;
+        while (size && *ptr != 0) {
+            size--; ptr++;
+        }
+        if (size == 0) {
+            where = __LINE__;
+            goto out;
+        }
+        size--; ptr++;
+        memmove(to, ptr, size);
     }
-    if (size == 0)
-	return -7;
-    size--; ptr++;
-
-    memmove(to, ptr, size);
 
  out:
     mp_clear_multi(&e, &n, &in, &out, &b, &bi, NULL);
-
-    return size;
+    return (ret == MP_OKAY) ? size : -where;
 }
 
 static BIGNUM *
@@ -496,66 +452,109 @@ mpz2BN(mp_int *s)
 {
     size_t size;
     BIGNUM *bn;
+    mp_err ret;
     void *p;
 
-    size = mp_unsigned_bin_size(s);
-    p = malloc(size);
-    if (p == NULL && size != 0)
+    size = mp_ubin_size(s);
+    if (size == 0)
 	return NULL;
 
-    mp_to_unsigned_bin(s, p);
+    p = malloc(size);
+    if (p == NULL)
+	return NULL;
 
-    bn = BN_bin2bn(p, size, NULL);
+    ret = mp_to_ubin(s, p, SIZE_MAX, NULL);
+    if (ret == MP_OKAY)
+        bn = BN_bin2bn(p, size, NULL);
     free(p);
-    return bn;
+    return (ret == MP_OKAY) ? bn : NULL;
+}
+
+enum gen_pq_type { GEN_P, GEN_Q };
+
+static int
+gen_p(int bits, enum gen_pq_type pq_type, uint8_t nibble_pair, mp_int *p, mp_int *e, BN_GENCB *cb)
+{
+    unsigned char *buf = NULL;
+    mp_bool res;
+    mp_err ret = MP_MEM;
+    mp_int t1, t2;
+    size_t len = (bits + 7) / 8;
+    int trials = mp_prime_rabin_miller_trials(bits);
+    int counter = 0;
+    int where HEIMDAL_UNUSED_ATTRIBUTE = 0;
+
+
+    FIRST(mp_init_multi(&t1, &t2, NULL));
+    if (ret == MP_OKAY && (buf = malloc(len))) do {
+        BN_GENCB_call(cb, 2, counter++);
+        /* random bytes */
+        ret = (RAND_bytes(buf, len) == 1) ? MP_OKAY : MP_ERR;
+
+        /* make it odd */
+        buf[len - 1] |= 1;
+
+        /* ensure the high nibble of the product is at least 128 */
+        if (pq_type == GEN_P)
+            buf[0] = (nibble_pair & 0xf0)         | (buf[0] & 0x0f);
+        else
+            buf[0] = ((nibble_pair & 0x0f) << 4)  | (buf[0] & 0x0f);
+
+        /* load number */
+        THEN_MP(mp_from_ubin(p, buf, len));
+
+        /* test primality; repeat if not */
+        THEN_MP(mp_prime_is_prime(p, trials, &res));
+        if (ret == MP_OKAY && res == MP_NO) continue;
+
+        /* check gcd(p - 1, e) == 1 */
+	THEN_MP(mp_sub_d(p, 1, &t1));
+	THEN_MP(mp_gcd(&t1, e, &t2));
+    } while (ret == MP_OKAY && mp_cmp_d(&t2, 1) != MP_EQ);
+
+    mp_clear_multi(&t1, &t2, NULL);
+    free(buf);
+    return ret;
 }
 
-#define CHECK(f, v) if ((f) != (v)) { goto out; }
+static uint8_t pq_high_nibble_pairs[] = {
+0x9f, 0xad, 0xae, 0xaf, 0xbc, 0xbd, 0xbe, 0xbf, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf9,
+0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff
+};
 
 static int
 ltm_rsa_generate_key(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb)
 {
     mp_int el, p, q, n, d, dmp1, dmq1, iqmp, t1, t2, t3;
-    int counter, ret, bitsp;
+    mp_err ret;
+    uint8_t high_nibbles = 0;
+    int bitsp;
+    int where = __LINE__;
 
     if (bits < 789)
 	return -1;
 
     bitsp = (bits + 1) / 2;
 
-    ret = -1;
-
-    mp_init_multi(&el, &p, &q, &n, &d,
-		  &dmp1, &dmq1, &iqmp,
-		  &t1, &t2, &t3, NULL);
+    FIRST(mp_init_multi(&el, &p, &q, &n, &d,
+                        &dmp1, &dmq1, &iqmp,
+                        &t1, &t2, &t3, NULL));
+    THEN_MP(BN2mpz(&el, e));
 
-    BN2mpz(&el, e);
+    /*
+     * randomly pick a pair of high nibbles for p and q to ensure the product's
+     * high nibble is at least 128
+     */
+    if (ret == MP_OKAY)
+        ret = (RAND_bytes(&high_nibbles, 1) == 1) ? MP_OKAY : MP_ERR;
+    high_nibbles %= sizeof(pq_high_nibble_pairs);
+    high_nibbles = pq_high_nibble_pairs[high_nibbles];
 
     /* generate p and q so that p != q and bits(pq) ~ bits */
-    counter = 0;
-    do {
-	BN_GENCB_call(cb, 2, counter++);
-	CHECK(random_num(&p, bitsp), 0);
-	CHECK(mp_find_prime(&p,128), MP_YES);
-
-	mp_sub_d(&p, 1, &t1);
-	mp_gcd(&t1, &el, &t2);
-    } while(mp_cmp_d(&t2, 1) != 0);
-
+    THEN_MP(gen_p(bitsp, GEN_P, high_nibbles, &p, &el, cb));
     BN_GENCB_call(cb, 3, 0);
-
-    counter = 0;
-    do {
-	BN_GENCB_call(cb, 2, counter++);
-	CHECK(random_num(&q, bits - bitsp), 0);
-	CHECK(mp_find_prime(&q,128), MP_YES);
-
-	if (mp_cmp(&p, &q) == 0) /* don't let p and q be the same */
-	    continue;
-
-	mp_sub_d(&q, 1, &t1);
-	mp_gcd(&t1, &el, &t2);
-    } while(mp_cmp_d(&t2, 1) != 0);
+    THEN_MP(gen_p(bitsp, GEN_Q, high_nibbles, &q, &el, cb));
 
     /* make p > q */
     if (mp_cmp(&p, &q) < 0) {
@@ -568,40 +567,38 @@ ltm_rsa_generate_key(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb)
     BN_GENCB_call(cb, 3, 1);
 
     /* calculate n,  		n = p * q */
-    mp_mul(&p, &q, &n);
+    THEN_MP(mp_mul(&p, &q, &n));
 
     /* calculate d, 		d = 1/e mod (p - 1)(q - 1) */
-    mp_sub_d(&p, 1, &t1);
-    mp_sub_d(&q, 1, &t2);
-    mp_mul(&t1, &t2, &t3);
-    mp_invmod(&el, &t3, &d);
+    THEN_MP(mp_sub_d(&p, 1, &t1));
+    THEN_MP(mp_sub_d(&q, 1, &t2));
+    THEN_MP(mp_mul(&t1, &t2, &t3));
+    THEN_MP(mp_invmod(&el, &t3, &d));
 
     /* calculate dmp1		dmp1 = d mod (p-1) */
-    mp_mod(&d, &t1, &dmp1);
+    THEN_MP(mp_mod(&d, &t1, &dmp1));
     /* calculate dmq1		dmq1 = d mod (q-1) */
-    mp_mod(&d, &t2, &dmq1);
+    THEN_MP(mp_mod(&d, &t2, &dmq1));
     /* calculate iqmp 		iqmp = 1/q mod p */
-    mp_invmod(&q, &p, &iqmp);
+    THEN_MP(mp_invmod(&q, &p, &iqmp));
 
     /* fill in RSA key */
 
-    rsa->e = mpz2BN(&el);
-    rsa->p = mpz2BN(&p);
-    rsa->q = mpz2BN(&q);
-    rsa->n = mpz2BN(&n);
-    rsa->d = mpz2BN(&d);
-    rsa->dmp1 = mpz2BN(&dmp1);
-    rsa->dmq1 = mpz2BN(&dmq1);
-    rsa->iqmp = mpz2BN(&iqmp);
-
-    ret = 1;
+    if (ret == MP_OKAY) {
+        rsa->e = mpz2BN(&el);
+        rsa->p = mpz2BN(&p);
+        rsa->q = mpz2BN(&q);
+        rsa->n = mpz2BN(&n);
+        rsa->d = mpz2BN(&d);
+        rsa->dmp1 = mpz2BN(&dmp1);
+        rsa->dmq1 = mpz2BN(&dmq1);
+        rsa->iqmp = mpz2BN(&iqmp);
+    }
 
-out:
     mp_clear_multi(&el, &p, &q, &n, &d,
 		   &dmp1, &dmq1, &iqmp,
 		   &t1, &t2, &t3, NULL);
-
-    return ret;
+    return (ret == MP_OKAY) ? 1 : -where;
 }
 
 static int