From cb9de77345640af920dd6aa54b583d0d339cbaf1 Mon Sep 17 00:00:00 2001 From: Kostas Tsiounis Date: Thu, 5 Mar 2026 13:07:49 -0500 Subject: [PATCH] Check that integer representation of message is smaller than modulus In RSA cipher operations, the message is converted into an integer that needs to be smaller than the public key's modulus. A check is added to fail early with the proper message, instead of failing during the actual operation with a not so helpful message. Additional tests to verify that behaviour are introduced as well. Signed-off-by: Kostas Tsiounis --- .../com/ibm/crypto/plus/provider/RSA.java | 29 +++++++---- .../ibm/jceplus/junit/tests/BaseTestRSA.java | 50 +++++++++++++++++++ 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/ibm/crypto/plus/provider/RSA.java b/src/main/java/com/ibm/crypto/plus/provider/RSA.java index 87b6836bf..474fcdfbc 100644 --- a/src/main/java/com/ibm/crypto/plus/provider/RSA.java +++ b/src/main/java/com/ibm/crypto/plus/provider/RSA.java @@ -11,6 +11,7 @@ import com.ibm.crypto.plus.provider.base.NativeException; import com.ibm.crypto.plus.provider.base.RSACipher; import com.ibm.crypto.plus.provider.base.RSAPadding; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.security.AlgorithmParameters; import java.security.InvalidAlgorithmParameterException; @@ -45,6 +46,7 @@ public final class RSA extends CipherSpi { private int keyType = -1; private boolean initialized = false; private boolean encrypting = true; + private RSAPublicKey rsaPub = null; private static final boolean doTypeChecking; private static final String DO_TYPE_CHECKING = "com.ibm.crypto.provider.DoRSATypeChecking"; @@ -104,14 +106,21 @@ protected int engineDoFinal(byte[] input, int inOffset, int inLen, byte[] output try { int outLen = 0; if (this.encrypting) { - if (this.padding.isPadding(RSAPadding.RSAPAD_NONE) - && msgLength != engineGetOutputSize(0)) { - byte[] paddedInput = new byte[engineGetOutputSize(0)]; - System.arraycopy(this.msgBuffer.array(), 0, paddedInput, - paddedInput.length - msgLength, msgLength); - this.msgBuffer.clear(); - this.msgBuffer.put(paddedInput); - this.msgLength = paddedInput.length; + if (this.padding.isPadding(RSAPadding.RSAPAD_NONE)) { + if (msgLength != engineGetOutputSize(0)) { + byte[] paddedInput = new byte[engineGetOutputSize(0)]; + System.arraycopy(this.msgBuffer.array(), 0, paddedInput, + paddedInput.length - msgLength, msgLength); + this.msgBuffer.clear(); + this.msgBuffer.put(paddedInput); + this.msgLength = paddedInput.length; + } + + BigInteger m = new BigInteger(1, this.msgBuffer.array()); + BigInteger n = this.rsaPub.getModulus(); + if (m.compareTo(n) >= 0) { + throw new BadPaddingException("Message is larger than modulus"); + } } else if (this.padding.isPadding(RSAPadding.RSAPAD_PKCS1) && msgLength > pkcs1InputLimit()) { throw new IllegalBlockSizeException( @@ -286,7 +295,7 @@ private void internalInit(int opmode, Key key, AlgorithmParameterSpec params) } } try { - RSAPublicKey rsaPub = (RSAPublicKey) rsaKey; + this.rsaPub = (RSAPublicKey) rsaKey; rsaCipher.initialize(rsaPub.getOCKKey(), false); this.keyType = Cipher.PUBLIC_KEY; } catch (Exception e) { @@ -300,6 +309,7 @@ private void internalInit(int opmode, Key key, AlgorithmParameterSpec params) } try { RSAPrivateCrtKey rsaPriv = (RSAPrivateCrtKey) rsaKey; + this.rsaPub = null; rsaCipher.initialize(rsaPriv.getOCKKey(), false); this.keyType = Cipher.PRIVATE_KEY; } catch (Exception e) { @@ -313,6 +323,7 @@ private void internalInit(int opmode, Key key, AlgorithmParameterSpec params) } try { RSAPrivateKey rsaPriv = (RSAPrivateKey) rsaKey; + this.rsaPub = null; rsaCipher.initialize(rsaPriv.getOCKKey(), true); this.keyType = Cipher.PRIVATE_KEY; } catch (Exception e) { diff --git a/src/test/java/ibm/jceplus/junit/tests/BaseTestRSA.java b/src/test/java/ibm/jceplus/junit/tests/BaseTestRSA.java index caafb409c..0261acb7d 100644 --- a/src/test/java/ibm/jceplus/junit/tests/BaseTestRSA.java +++ b/src/test/java/ibm/jceplus/junit/tests/BaseTestRSA.java @@ -637,6 +637,56 @@ public void testRSACipherExceedInput() throws Exception { } } + @Test + public void testRSACipherNoPaddingExceedInput() throws Exception { + // FIPS does not support non-OAEP paddings. + assumeFalse("OpenJCEPlusFIPS".equals(getProviderName())); + + rsaKeyPairGen.initialize(2048); + KeyPair rsaKeyPair = rsaKeyPairGen.generateKeyPair(); + RSAPublicKey pubKey = (RSAPublicKey) rsaKeyPair.getPublic(); + Cipher cp = Cipher.getInstance("RSA/ECB/NoPadding", getProviderName()); + + BigInteger modulus = ((RSAKey) pubKey).getModulus(); + byte[] modulusPlusOne = modulus.add(BigInteger.ONE).toByteArray(); + byte[] plaintext = Arrays.copyOfRange(modulusPlusOne, 1, modulusPlusOne.length); // BigInteger has an extra byte for sign. + try { + cp.init(Cipher.ENCRYPT_MODE, pubKey); + cp.doFinal(plaintext); + + fail("Did not get expected BadPaddingException."); + } catch (BadPaddingException bpe) { + assertEquals("Message is larger than modulus", bpe.getMessage(), "Exception message is not what's expected."); + } + + byte[] modulusArray = modulus.toByteArray(); + plaintext = Arrays.copyOfRange(modulusArray, 1, modulusArray.length); // BigInteger has an extra byte for sign. + try { + cp.init(Cipher.ENCRYPT_MODE, pubKey); + cp.doFinal(plaintext); + + fail("Did not get expected BadPaddingException."); + } catch (BadPaddingException bpe) { + assertEquals("Message is larger than modulus", bpe.getMessage(), "Exception message is not what's expected."); + } + } + + @Test + public void testRSACipherSmallMessageLargeFirstByte() throws Exception { + // FIPS does not support non-OAEP paddings. + assumeFalse("OpenJCEPlusFIPS".equals(getProviderName())); + + rsaKeyPairGen.initialize(2048); + KeyPair rsaKeyPair = rsaKeyPairGen.generateKeyPair(); + RSAPublicKey pubKey = (RSAPublicKey) rsaKeyPair.getPublic(); + Cipher cp = Cipher.getInstance("RSA/ECB/NoPadding", getProviderName()); + cp.init(Cipher.ENCRYPT_MODE, pubKey); + + byte[] plaintext = {(byte) 0xFF, (byte) 0x01, (byte) 0x01, (byte) 0x01, (byte) 0x01}; + + cp.doFinal(plaintext); + } + @Test public void testRSACipher_init_cert() throws Exception { // FIXME