Skip to content

Check that integer representation of message is smaller than modulus #1252

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
KostasTsiounis wants to merge 1 commit into
base: main
Choose a base branch
from
+70 −9
Open

Check that integer representation of message is smaller than modulus #1252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions src/main/java/com/ibm/crypto/plus/provider/RSA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import com.ibm.crypto.plus.provider.base.NativeException;
import com.ibm.crypto.plus.provider.base.RSACipher;
import com.ibm.crypto.plus.provider.base.RSAPadding;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.security.AlgorithmParameters;
import java.security.InvalidAlgorithmParameterException;
Expand Down Expand Up @@ -45,6 +46,7 @@ public final class RSA extends CipherSpi {
private int keyType = -1;
private boolean initialized = false;
private boolean encrypting = true;
private RSAPublicKey rsaPub = null;

private static final boolean doTypeChecking;
private static final String DO_TYPE_CHECKING = "com.ibm.crypto.provider.DoRSATypeChecking";
Expand Down Expand Up @@ -104,14 +106,21 @@ protected int engineDoFinal(byte[] input, int inOffset, int inLen, byte[] output
try {
int outLen = 0;
if (this.encrypting) {
Comment thread
jasonkatonica marked this conversation as resolved.
if (this.padding.isPadding(RSAPadding.RSAPAD_NONE)
&& msgLength != engineGetOutputSize(0)) {
byte[] paddedInput = new byte[engineGetOutputSize(0)];
System.arraycopy(this.msgBuffer.array(), 0, paddedInput,
paddedInput.length - msgLength, msgLength);
this.msgBuffer.clear();
this.msgBuffer.put(paddedInput);
this.msgLength = paddedInput.length;
if (this.padding.isPadding(RSAPadding.RSAPAD_NONE)) {
if (msgLength != engineGetOutputSize(0)) {
byte[] paddedInput = new byte[engineGetOutputSize(0)];
System.arraycopy(this.msgBuffer.array(), 0, paddedInput,
paddedInput.length - msgLength, msgLength);
this.msgBuffer.clear();
this.msgBuffer.put(paddedInput);
this.msgLength = paddedInput.length;
}

BigInteger m = new BigInteger(1, this.msgBuffer.array());
BigInteger n = this.rsaPub.getModulus();
if (m.compareTo(n) >= 0) {
throw new BadPaddingException("Message is larger than modulus");
}
} else if (this.padding.isPadding(RSAPadding.RSAPAD_PKCS1)
&& msgLength > pkcs1InputLimit()) {
throw new IllegalBlockSizeException(
Expand Down Expand Up @@ -286,7 +295,7 @@ private void internalInit(int opmode, Key key, AlgorithmParameterSpec params)
}
}
try {
RSAPublicKey rsaPub = (RSAPublicKey) rsaKey;
this.rsaPub = (RSAPublicKey) rsaKey;
rsaCipher.initialize(rsaPub.getOCKKey(), false);
this.keyType = Cipher.PUBLIC_KEY;
} catch (Exception e) {
Expand All @@ -300,6 +309,7 @@ private void internalInit(int opmode, Key key, AlgorithmParameterSpec params)
}
try {
RSAPrivateCrtKey rsaPriv = (RSAPrivateCrtKey) rsaKey;
this.rsaPub = null;
rsaCipher.initialize(rsaPriv.getOCKKey(), false);
this.keyType = Cipher.PRIVATE_KEY;
} catch (Exception e) {
Expand All @@ -313,6 +323,7 @@ private void internalInit(int opmode, Key key, AlgorithmParameterSpec params)
}
try {
RSAPrivateKey rsaPriv = (RSAPrivateKey) rsaKey;
this.rsaPub = null;
rsaCipher.initialize(rsaPriv.getOCKKey(), true);
this.keyType = Cipher.PRIVATE_KEY;
} catch (Exception e) {
Expand Down
50 changes: 50 additions & 0 deletions src/test/java/ibm/jceplus/junit/tests/BaseTestRSA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -637,6 +637,56 @@ public void testRSACipherExceedInput() throws Exception {
}
}

@Test
public void testRSACipherNoPaddingExceedInput() throws Exception {
// FIPS does not support non-OAEP paddings.
assumeFalse("OpenJCEPlusFIPS".equals(getProviderName()));

rsaKeyPairGen.initialize(2048);
KeyPair rsaKeyPair = rsaKeyPairGen.generateKeyPair();
RSAPublicKey pubKey = (RSAPublicKey) rsaKeyPair.getPublic();
Cipher cp = Cipher.getInstance("RSA/ECB/NoPadding", getProviderName());

BigInteger modulus = ((RSAKey) pubKey).getModulus();
byte[] modulusPlusOne = modulus.add(BigInteger.ONE).toByteArray();
byte[] plaintext = Arrays.copyOfRange(modulusPlusOne, 1, modulusPlusOne.length); // BigInteger has an extra byte for sign.
try {
cp.init(Cipher.ENCRYPT_MODE, pubKey);
cp.doFinal(plaintext);

fail("Did not get expected BadPaddingException.");
} catch (BadPaddingException bpe) {
assertEquals("Message is larger than modulus", bpe.getMessage(), "Exception message is not what's expected.");
}

byte[] modulusArray = modulus.toByteArray();
plaintext = Arrays.copyOfRange(modulusArray, 1, modulusArray.length); // BigInteger has an extra byte for sign.
try {
cp.init(Cipher.ENCRYPT_MODE, pubKey);
cp.doFinal(plaintext);

fail("Did not get expected BadPaddingException.");
} catch (BadPaddingException bpe) {
assertEquals("Message is larger than modulus", bpe.getMessage(), "Exception message is not what's expected.");
}
}

@Test
public void testRSACipherSmallMessageLargeFirstByte() throws Exception {
// FIPS does not support non-OAEP paddings.
assumeFalse("OpenJCEPlusFIPS".equals(getProviderName()));

rsaKeyPairGen.initialize(2048);
KeyPair rsaKeyPair = rsaKeyPairGen.generateKeyPair();
RSAPublicKey pubKey = (RSAPublicKey) rsaKeyPair.getPublic();
Cipher cp = Cipher.getInstance("RSA/ECB/NoPadding", getProviderName());
cp.init(Cipher.ENCRYPT_MODE, pubKey);

byte[] plaintext = {(byte) 0xFF, (byte) 0x01, (byte) 0x01, (byte) 0x01, (byte) 0x01};

cp.doFinal(plaintext);
}

@Test
public void testRSACipher_init_cert() throws Exception {
// FIXME
Expand Down
Loading