~aritra1911/openssl_madness

b96aa0ad3a60ef5637985805ec8aa8a1adee7492 — Aritra Sarkar 2 years ago d56ad0a
Avoid OpenSSL 3.0 deprecations during RSA encrypt
2 files changed, 53 insertions(+), 14 deletions(-)

M madness.c
M notes.txt
M madness.c => madness.c +46 -14
@@ 6,6 6,7 @@
#include <openssl/pem.h>
#include <openssl/evp.h>
#include <openssl/rsa.h>
#include <openssl/engine.h>
#include <openssl/err.h>

int rsa_encrypt(EVP_PKEY*, FILE*, FILE*, const int);


@@ 236,15 237,44 @@ int main(int argc, char* argv[]) {
}

int rsa_encrypt(EVP_PKEY* public_key, FILE* infile, FILE* outfile, const int verbose) {
    int len, key_size = EVP_PKEY_get_size(public_key);
    char *buf, *encbuf;
    size_t flen;
    int key_size = EVP_PKEY_get_size(public_key);
    size_t flen, len;
    unsigned char *buf, *encbuf;
    EVP_PKEY_CTX* ctx;

    /* Get default RSA context from `public_key` */
    if ( !(ctx = EVP_PKEY_CTX_new(public_key, NULL)) ) {
        ERR_print_errors_fp(stderr);
        return -1;
    }

    /* Initialize the context for encryption */
    if ( EVP_PKEY_encrypt_init(ctx) <= 0 ) {
        ERR_print_errors_fp(stderr);
        return -1;
    }

    /* Set padding type for context */
    if ( EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0 ) {
        ERR_print_errors_fp(stderr);
        return -1;
    }

    // Quoting from man page of RSA_PUBLIC_ENCRYPT(3):
    // "flen must not be more than RSA_size(rsa) - 42 for RSA_PKCS1_OAEP_PADDING"
    buf = malloc(key_size - 42);
    if ( !(buf = OPENSSL_malloc(key_size - 42)) ) {
        fprintf(stderr, "Failed to allocate memory for input buffer\n");
        EVP_PKEY_CTX_free(ctx);
        return -1;
    }

    // Encrypted block with padding will be `key_size' bytes long
    encbuf = malloc(key_size);
    if ( !(encbuf = OPENSSL_malloc(key_size)) ) {
        fprintf(stderr, "Failed to allocate memory!\n");
        OPENSSL_free(buf);
        EVP_PKEY_CTX_free(ctx);
        return -1;
    }

    do {
        // Read blocks of data from `infile' and encrypt them and write them out to `outfile', one block at a time.


@@ 254,25 284,27 @@ int rsa_encrypt(EVP_PKEY* public_key, FILE* infile, FILE* outfile, const int ver
        // "If the end of the file is reached, the return value is a short item count (or zero)"
        // Hence, if 0 bytes were read, that's definitely an EOF, implying that we must stop
        if (!(flen = fread(buf, 1, key_size - 42, infile))) break;
        if (verbose) printf("%lu bytes read for encryption\n", flen);
        if (verbose) printf("INFO : %lu bytes read for encryption\n", flen);

        if ((len = RSA_public_encrypt(flen, (unsigned char*) buf, (unsigned char*) encbuf,
                                      public_key, RSA_PKCS1_OAEP_PADDING)) != key_size) {
        /* Now encrypt `flen` bytes which should produce `len` bytes */
        if ( EVP_PKEY_encrypt(ctx, encbuf, &len, buf, flen) <= 0 ) {
            fprintf(stderr, " ERR : ");
            ERR_print_errors_fp(stderr);
            free(buf);
            free(encbuf);
            return -1;
        }
        if (verbose) printf("%d bytes encrypted\n", len);

        if (verbose) printf("INFO : %lu bytes encrypted\n", flen);

        // Write it to outfile
        flen = fwrite(encbuf, 1, len, outfile);
        if (verbose) printf("%lu bytes written\n", flen);
        len = fwrite(encbuf, 1, len, outfile);
        if (verbose) printf("INFO : %lu bytes written\n", len);

    } while (!feof(infile));

    free(buf);
    free(encbuf);
    OPENSSL_free(buf);
    OPENSSL_free(encbuf);
    EVP_PKEY_CTX_free(ctx);

    return 0;
}

M notes.txt => notes.txt +7 -0
@@ 25,3 25,10 @@ Hello, World!

    I’ve got no idea where the PKCS#1 vs SubjectPublicKeyInfo comes into it when I’m just trying to load that file. Regardless, the only one that
    works is PEM_read_RSA_PUBKEY()

#################
## OpenSSL 3.0 ##
#################

# Test whether encryption works or not.
$ fortune | ./madness -e -k pubkey.pem | /opt/openssl-3.0.0-beta1/bin/openssl pkeyutl -decrypt -inkey key.pem -pkeyopt rsa_padding_mode:oaep