[patch] fix to compile with a new openssl

dmg2img-opessl-1.1-fix.patch (text/plain), 6.91 KB, created by milan hodoscek on 2018-12-31 11:51:24 UTC

diff --git a/old/vfdecrypt.c b/vfdecrypt.c
index 56d3530..b1a36d3 100644
--- a/old/vfdecrypt.c
+++ b/vfdecrypt.c
@@ -183,7 +183,7 @@ void adjust_v2_header_byteorder(cencrypted_v2_pwheader *pwhdr) {
   pwhdr->encrypted_keyblob_size = htonl(pwhdr->encrypted_keyblob_size);
 }
 
-HMAC_CTX hmacsha1_ctx;
+HMAC_CTX *hmacsha1_ctx;
 AES_KEY aes_decrypt_key;
 int CHUNK_SIZE=4096;  // default
 
@@ -196,9 +196,9 @@ void compute_iv(uint32_t chunk_no, uint8_t *iv) {
   unsigned int mdLen;
   
   chunk_no = OSSwapHostToBigInt32(chunk_no);
-  HMAC_Init_ex(&hmacsha1_ctx, NULL, 0, NULL, NULL);
-  HMAC_Update(&hmacsha1_ctx, (void *) &chunk_no, sizeof(uint32_t));
-  HMAC_Final(&hmacsha1_ctx, mdResult, &mdLen);
+  HMAC_Init_ex(hmacsha1_ctx, NULL, 0, NULL, NULL);
+  HMAC_Update(hmacsha1_ctx, (void *) &chunk_no, sizeof(uint32_t));
+  HMAC_Final(hmacsha1_ctx, mdResult, &mdLen);
   memcpy(iv, mdResult, CIPHER_BLOCKSIZE);
 }
 
@@ -212,52 +212,75 @@ void decrypt_chunk(uint8_t *ctext, uint8_t *ptext, uint32_t chunk_no) {
 /* DES3-EDE unwrap operation loosely based on to RFC 2630, section 12.6 
  *    wrapped_key has to be 40 bytes in length.  */
 int apple_des3_ede_unwrap_key(uint8_t *wrapped_key, int wrapped_key_len, uint8_t *decryptKey, uint8_t *unwrapped_key) {
-  EVP_CIPHER_CTX ctx;
+  EVP_CIPHER_CTX *ctx;
   uint8_t *TEMP1, *TEMP2, *CEKICV;
   uint8_t IV[8] = { 0x4a, 0xdd, 0xa2, 0x2c, 0x79, 0xe8, 0x21, 0x05 };
   int outlen, tmplen, i;
 
-  EVP_CIPHER_CTX_init(&ctx);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+  ctx = EVP_CIPHER_CTX_new();
+#else
+  ctx = malloc(sizeof(*ctx));
+#endif
+  if (!ctx) {
+    fprintf(stderr, "Out of memory: EVP_CIPHER_CTX!\n");
+    return(-1);
+  }
+
+  EVP_CIPHER_CTX_init(ctx);
   /* result of the decryption operation shouldn't be bigger than ciphertext */
   TEMP1 = malloc(wrapped_key_len);
   TEMP2 = malloc(wrapped_key_len);
   CEKICV = malloc(wrapped_key_len);
   /* uses PKCS#7 padding for symmetric key operations by default */
-  EVP_DecryptInit_ex(&ctx, EVP_des_ede3_cbc(), NULL, decryptKey, IV);
+  EVP_DecryptInit_ex(ctx, EVP_des_ede3_cbc(), NULL, decryptKey, IV);
 
-  if(!EVP_DecryptUpdate(&ctx, TEMP1, &outlen, wrapped_key, wrapped_key_len)) {
+  if(!EVP_DecryptUpdate(ctx, TEMP1, &outlen, wrapped_key, wrapped_key_len)) {
     fprintf(stderr, "internal error (1) during key unwrap operation!\n");
     return(-1);
   }
-  if(!EVP_DecryptFinal_ex(&ctx, TEMP1 + outlen, &tmplen)) {
+  if(!EVP_DecryptFinal_ex(ctx, TEMP1 + outlen, &tmplen)) {
     fprintf(stderr, "internal error (2) during key unwrap operation!\n");
     return(-1);
   }
   outlen += tmplen;
-  EVP_CIPHER_CTX_cleanup(&ctx);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+  EVP_CIPHER_CTX_reset(ctx);
+#else
+  EVP_CIPHER_CTX_cleanup(ctx);
+#endif
 
   /* reverse order of TEMP3 */
   for(i = 0; i < outlen; i++) TEMP2[i] = TEMP1[outlen - i - 1];
 
-  EVP_CIPHER_CTX_init(&ctx);
+  EVP_CIPHER_CTX_init(ctx);
   /* uses PKCS#7 padding for symmetric key operations by default */
-  EVP_DecryptInit_ex(&ctx, EVP_des_ede3_cbc(), NULL, decryptKey, TEMP2);
-  if(!EVP_DecryptUpdate(&ctx, CEKICV, &outlen, TEMP2+8, outlen-8)) {
+  EVP_DecryptInit_ex(ctx, EVP_des_ede3_cbc(), NULL, decryptKey, TEMP2);
+  if(!EVP_DecryptUpdate(ctx, CEKICV, &outlen, TEMP2+8, outlen-8)) {
     fprintf(stderr, "internal error (3) during key unwrap operation!\n");
     return(-1);
   }
-  if(!EVP_DecryptFinal_ex(&ctx, CEKICV + outlen, &tmplen)) {
+  if(!EVP_DecryptFinal_ex(ctx, CEKICV + outlen, &tmplen)) {
     fprintf(stderr, "internal error (4) during key unwrap operation!\n");
     return(-1);
   }
 
   outlen += tmplen;
-  EVP_CIPHER_CTX_cleanup(&ctx);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+  EVP_CIPHER_CTX_reset(ctx);
+#else
+  EVP_CIPHER_CTX_cleanup(ctx);
+#endif
 
   memcpy(unwrapped_key, CEKICV+4, outlen-4);
   free(TEMP1);
   free(TEMP2);
   free(CEKICV);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+  EVP_CIPHER_CTX_free(ctx);
+#else
+  free(ctx);
+#endif
   return(0);
 }
 
@@ -279,31 +302,46 @@ int unwrap_v1_header(char *passphrase, cencrypted_v1_header *header, uint8_t *ae
 int unwrap_v2_header(char *passphrase, cencrypted_v2_pwheader *header, uint8_t *aes_key, uint8_t *hmacsha1_key) {
   /* derived key is a 3DES-EDE key */
   uint8_t derived_key[192/8];
-  EVP_CIPHER_CTX ctx;
+  EVP_CIPHER_CTX *ctx;
   uint8_t *TEMP1;
   int outlen, tmplen;
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+  ctx = EVP_CIPHER_CTX_new();
+#else
+  ctx = malloc(sizeof(*ctx));
+#endif
+  if (!ctx) {
+    fprintf(stderr, "Out of memory: EVP_CIPHER_CTX!\n");
+    return(-1);
+  }
+
   PKCS5_PBKDF2_HMAC_SHA1(passphrase, strlen(passphrase), (unsigned char*)header->kdf_salt, 20,
 			 PBKDF2_ITERATION_COUNT, sizeof(derived_key), derived_key);
 
   print_hex(derived_key, 192/8);
 
-  EVP_CIPHER_CTX_init(&ctx);
+  EVP_CIPHER_CTX_init(ctx);
   /* result of the decryption operation shouldn't be bigger than ciphertext */
   TEMP1 = malloc(header->encrypted_keyblob_size);
   /* uses PKCS#7 padding for symmetric key operations by default */
-  EVP_DecryptInit_ex(&ctx, EVP_des_ede3_cbc(), NULL, derived_key, header->blob_enc_iv);
+  EVP_DecryptInit_ex(ctx, EVP_des_ede3_cbc(), NULL, derived_key, header->blob_enc_iv);
 
-  if(!EVP_DecryptUpdate(&ctx, TEMP1, &outlen, header->encrypted_keyblob, header->encrypted_keyblob_size)) {
+  if(!EVP_DecryptUpdate(ctx, TEMP1, &outlen, header->encrypted_keyblob, header->encrypted_keyblob_size)) {
     fprintf(stderr, "internal error (1) during key unwrap operation!\n");
     return(-1);
   }
-  if(!EVP_DecryptFinal_ex(&ctx, TEMP1 + outlen, &tmplen)) {
+  if(!EVP_DecryptFinal_ex(ctx, TEMP1 + outlen, &tmplen)) {
     fprintf(stderr, "internal error (2) during key unwrap operation!\n");
     return(-1);
   }
   outlen += tmplen;
-  EVP_CIPHER_CTX_cleanup(&ctx);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+  EVP_CIPHER_CTX_free(ctx);
+#else
+  EVP_CIPHER_CTX_cleanup(ctx);
+  free(ctx);
+#endif
   memcpy(aes_key, TEMP1, 16);
   memcpy(hmacsha1_key, TEMP1, 20);
 
@@ -446,8 +484,21 @@ int main(int argc, char *argv[]) {
     CHUNK_SIZE = v2header.blocksize;
   }
   
-  HMAC_CTX_init(&hmacsha1_ctx);
-  HMAC_Init_ex(&hmacsha1_ctx, hmacsha1_key, sizeof(hmacsha1_key), EVP_sha1(), NULL);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+  hmacsha1_ctx = HMAC_CTX_new();
+#else
+  hmacsha1_ctx = malloc(sizeof(*hmacsha1_ctx));
+#endif
+  if (!hmacsha1_ctx) {
+    fprintf(stderr, "Out of memory: HMAC CTX!\n");
+    exit(1);
+  }
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+  HMAC_CTX_reset(hmacsha1_ctx);
+#else
+  HMAC_CTX_init(hmacsha1_ctx);
+#endif
+  HMAC_Init_ex(hmacsha1_ctx, hmacsha1_key, sizeof(hmacsha1_key), EVP_sha1(), NULL);
   AES_set_decrypt_key(aes_key, CIPHER_KEY_LENGTH * 8, &aes_decrypt_key);
   
   if (verbose >= 1) {
@@ -472,5 +523,11 @@ int main(int argc, char *argv[]) {
   }
   
   if (verbose)  fprintf(stderr, "%"PRIX32" chunks written\n", chunk_no);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+  HMAC_CTX_free(hmacsha1_ctx);
+#else
+  HMAC_CTX_cleanup(hmacsha1_ctx);
+  free(hmacsha1_ctx);
+#endif
   return(0);
 }