使用Web Crypto API进行RSA-OAEP解密失败问题排查

背景操作 #

使用OpenSSL完成RSA密钥生成、加密解密的全流程，验证完全正常：

生成密钥 #

openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:2048
openssl rsa -pubout -in private_key.pem -out public_key.pem
openssl pkcs8 -topk8 -inform PEM -outform PEM -in private_key.pem -out private_key_pkcs8.pem -nocrypt

加密明文 #

echo -n "abcdef" | openssl rsautl -encrypt -pubin -inkey public_key.pem | openssl base64 > enc_simple.txt

OpenSSL解密验证 #

base64 -d enc_simple.txt > enc_simple.bin
openssl rsautl -decrypt -inkey private_key_pkcs8.pem -in enc_simple.bin -out decrypted_simple.txt

可成功得到明文abcdef，确认OpenSSL操作无问题。

问题现象 #

使用Web Crypto API复现解密流程时，私钥导入正常，但解密操作抛出无详细信息的OperationError。已通过od -An -vtu1 enc_simple.bin确认加密后的二进制数据，与代码中str2ab转换结果完全一致。

附Web Crypto API代码：

const encryptedData = `gLiN3BLWTxbwpE5bwcFiB7Y/nK0H4iz9lK0sehMIJtoAPibDPZ8EYp8EnzsGFlKC
+GXse6Ka5YC7UKYn7xUvfBbCDSY7RJ/J9oAzpCEHvCnFOmrsvUtNEhP4w3LTaKL/
qKmjQZaPlZNWCmrzrW3g2A6DUCqLx5EQwCcwME2WjYVhuMZfZOKqtp+uukJMUf3h
cC9J8QkPCk1NZogCad29b7q7JLN1uWEQzOUgB6BqJfAp8kL92dftOy2gpWoGYraI
YPU3Tmp12txMyUY0yKsr4c2dOzxwWUW+ZssVkc6ZGqFVufADziZDp2sfFaz8yBuI
qek7S0KXd6A7qWBYBrzzyw==`; //abcdef
const privKeyPEM=`-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDJiQlvlYbTqi/W
bLa7nzaF4Wz+ypQa01HD+UMHX25Uv3n6P69ucKs/OOdWbcenknAmSBK/VzgIsjlm
vak3y1LoYImMhm8R26Arv8Fv7jwQkbBxVnOn6yfTL2h1SRK+zFzWumfbl2vp7POG
s7CRQaK0HFPKroVlYHlinTlYXSlFqdaknH2ip1J2YPJ3O4w0/ME8gBZ0Mj1XkokB
Qc0Lk76BpvfTuBbzvLYI4bq0/jSoM6ANecqK+T/7/aSFxh2x8acJ/xzJg2H7I+de
pXZ1M9Mz0x94jWRfc9Q9qLgY6AuX7MUyb2YEs+vMxcim60fvktHAaXvn9vM0eauC
B+hpThohAgMBAAECggEAINub0zqAwe/EXuRYopxhqlBHkf77SKhdc2MnX4NanKyf
OYK6mnn6IZOoe/noDFUevc8QZ2vT8e8E1tBjT3px6PscUfH1F+dD5P4djp874cOv
Dbt3ndAELTVUhZLFYKA3HrdDiZTVfk0oozSWvAgEe/MGYkwz6YRiJgbWO6bsvLOa
5Ehhd7Ocxbs4B389ad3bVzRkk02/IRDEeO1z0+22Wv1NIe0Spzl6FVLspL6JdmEu
60rpShfwN7D0G4TQYYp/9Sh+zt0ThIF4kqQoURx5E2uQaDUAT206itMG2mUWYSR1
M37U+46kpWBz0jDaRFmqlySKrmkbx7kWXju8wP1ZAQKBgQDvlq4B3jAPd+Pu/Rxw
GORV5S8PdJrkQnx8BeAviGAVc2w6EMyRWusVzL95mpRS0t0E5xZtB2kLkVU/I08b
j+WqrGFqFp8niXYnJsien8nfLct0AhJVvVaPtt8lHP9SKniBOZc0owzFljnt+4/6
M4ifAiKvJYOqPSdGpJgvJq4TsQKBgQDXVxJCjUNInb4p0WWljxdcMpgSDPn53+BG
M/MrxVbTOvHhKsbNlglYxT/87J+8lFKS5Y9EkpqekCY1aGJ0PYfLBMowvNAwiP2s
89++PTuEO/hK16Xb8znZ7rUGdJdRUitBN8iVkWNLA2pzalcZkkHbIWQbDuJOPbdO
bo1u06U5cQKBgAgeliUQD5bmnD3kLAuMfGiAzNh8PieQLUHSvSc/OupfMALDwPsI
FsF1X+PSHka0SLM61aK6RpASy83I94xakxD2qJJ808X2PZ/UC6Z8ic3bcnKrA04O
jZlvPB643do+ADl45yvsfqlPjwUGqnlzN6UT4HMJFW42hlc5isLGT83xAoGBANT/
UxhxEfRp0wcaECjKeJjBkpmILFp0jynhiM3qzA7zZv0JissfdO9RbBGJHBczvtl+
J0/0kuv0OVbqgTfpBMBTZIsAuAzJ8+F2+AD8IDqT9uxQkcYVt0tRSc2w1VuioxZH
TyhiPoycPFcdADpS6MEPLi11c3NgqEf0IgFVZ0CBAoGBAMybGC+IImSDpZzLaSpz
KAD6Vyto7eK4pENH8bPcM2Hpy7aMGJkr6Q/NT34p/vSfFG2gm8ueILhvqQ8yB6h6
gtfHOEu7CC/8WL4Y/WehrpD+fTTSOd/C+JZJbXjHjJ55NX1JzgeCPWJP2QOyXE85
VLq+2V8bhsQxi4JylnH6suLB
-----END PRIVATE KEY-----`
const
    atob = (window || self)?.atob,
    subtle = (window || self).crypto.subtle,
    trimWs = (s) => s.replace(/\s*/g,""),
    trimHeader = (s) => s.replace(/-----(?:BEGIN|END) PRIVATE KEY-----/gi, "");
async function importPrivateKey (pemkey) {
    const binaryKey = atob(trimWs(trimHeader(pemkey)));
    const keyBuffer = new Uint8Array(binaryKey.length);
    for (let i = 0; i < binaryKey.length; i++) {
        keyBuffer[i] = binaryKey.charCodeAt(i);
    }
    return await subtle.importKey(
        "pkcs8",
        keyBuffer,
        { name: "RSA-OAEP", hash: "SHA-256"},
        false,
        ["decrypt"]
    );
}
async function decryptData (privateKey, encryptedData) {
      const decrypted = await subtle.decrypt(
          { name: "RSA-OAEP", hash: "SHA-256"},
          privateKey,
         str2ab(atob(trimWs(encryptedData)))
      );
      return (new TextDecoder('utf-8')).decode(decrypted);
}
function str2ab (binaryStr) {
    const len = binaryStr.length;
    const bytes = new Uint8Array(len);
    for (let i = 0; i < len; i++) {
        bytes[i] = binaryStr.charCodeAt(i); // Accurately map each char to a byte
    }
    return bytes; // Convert to ArrayBuffer
};


(async () => {
  try {
    const privKey = await importPrivateKey(privKeyPEM);
    console.log(privKey);
    const decrypted = await decryptData(privKey, encryptedData);
  } catch (err) {
    console.error("Err:", err, "Err.massage:", err.message);
  }
})()

失败原因 #

填充算法不匹配：

• OpenSSL的rsautl命令默认使用PKCS#1 v1.5填充（对应Web Crypto的RSA-PKCS1-v1_5算法）。
• 你的Web Crypto API代码中配置的是RSA-OAEP填充，两种填充格式不兼容，导致解密时验证失败，抛出OperationError。

解决方案 #

两种修正方式任选其一即可：

方式1：修改Web Crypto代码，使用PKCS#1 v1.5填充 #

调整密钥导入和解密的算法配置：

async function importPrivateKey (pemkey) {
    const binaryKey = atob(trimWs(trimHeader(pemkey)));
    const keyBuffer = new Uint8Array(binaryKey.length);
    for (let i = 0; i < binaryKey.length; i++) {
        keyBuffer[i] = binaryKey.charCodeAt(i);
    }
    return await subtle.importKey(
        "pkcs8",
        keyBuffer,
        { name: "RSA-PKCS1-v1_5", hash: "SHA-256"}, // 改为RSA-PKCS1-v1_5
        false,
        ["decrypt"]
    );
}
async function decryptData (privateKey, encryptedData) {
      const decrypted = await subtle.decrypt(
          { name: "RSA-PKCS1-v1_5" }, // 仅指定算法名称即可
          privateKey,
         str2ab(atob(trimWs(encryptedData)))
      );
      return (new TextDecoder('utf-8')).decode(decrypted);
}

方式2：修改OpenSSL加密命令，使用OAEP填充 #

加密时指定-oaep参数，匹配Web Crypto的RSA-OAEP算法：

echo -n "abcdef" | openssl rsautl -encrypt -pubin -inkey public_key.pem -oaep | openssl base64 > enc_simple.txt

此时原Web Crypto代码无需修改，即可正常解密。

内容的提问来源于stack exchange，提问作者ibrahim tanyalcin