Skip to content

Commit e2d3130

Browse files
committed
handle failure in encrypt_rfbdes() in callers
1 parent 042a816 commit e2d3130

File tree

4 files changed

+49
-16
lines changed

4 files changed

+49
-16
lines changed

include/rfb/rfbproto.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1557,7 +1557,7 @@ typedef union {
15571557
extern int rfbEncryptAndStorePasswd(char *passwd, char *fname);
15581558
extern char *rfbDecryptPasswdFromFile(char *fname);
15591559
extern void rfbRandomBytes(unsigned char *bytes);
1560-
extern void rfbEncryptBytes(unsigned char *bytes, char *passwd);
1560+
extern int rfbEncryptBytes(unsigned char *bytes, char *passwd);
15611561

15621562

15631563
#endif

src/common/vncauth.c

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,10 @@ rfbEncryptAndStorePasswd(char *passwd, char *fname)
102102

103103
/* Do encryption in-place - this way we overwrite our copy of the plaintext
104104
password */
105-
encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd));
105+
if (encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd)) == 0) {
106+
fclose(fp);
107+
return 1;
108+
}
106109

107110
for (i = 0; i < 8; i++) {
108111
putc(encryptedPasswd[i], fp);
@@ -180,7 +183,7 @@ rfbRandomBytes(unsigned char *bytes)
180183
* Encrypt CHALLENGESIZE bytes in memory using a password.
181184
*/
182185

183-
void
186+
int
184187
rfbEncryptBytes(unsigned char *bytes, char *passwd)
185188
{
186189
unsigned char key[8];
@@ -197,19 +200,27 @@ rfbEncryptBytes(unsigned char *bytes, char *passwd)
197200
}
198201
}
199202

200-
encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE);
203+
if (encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE) == 0) {
204+
return 1;
205+
}
206+
return 0;
201207
}
202208

203-
void
209+
int
204210
rfbEncryptBytes2(unsigned char *where, const int length, unsigned char *key) {
205211
int i, j, out_len;
206212
for (i = 0; i< 8; i++)
207213
where[i] ^= key[i];
208-
encrypt_rfbdes(where, &out_len, key, where, 8);
214+
if (encrypt_rfbdes(where, &out_len, key, where, 8) == 0) {
215+
return 1;
216+
}
209217
for (i = 8; i < length; i += 8) {
210218
for (j = 0; j < 8; j++) {
211219
where[i + j] ^= where[i + j - 8];
212220
}
213-
encrypt_rfbdes(where + i, &out_len, key, where + i, 8);
221+
if (encrypt_rfbdes(where + i, &out_len, key, where + i, 8) == 0) {
222+
return 1;
223+
}
214224
}
225+
return 0;
215226
}

src/libvncclient/rfbclient.c

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -404,8 +404,8 @@ rfbBool ConnectToRFBRepeater(rfbClient* client,const char *repeaterHost, int rep
404404
return TRUE;
405405
}
406406

407-
extern void rfbClientEncryptBytes(unsigned char* bytes, char* passwd);
408-
extern void rfbClientEncryptBytes2(unsigned char *where, const int length, unsigned char *key);
407+
extern int rfbClientEncryptBytes(unsigned char* bytes, char* passwd);
408+
extern int rfbClientEncryptBytes2(unsigned char *where, const int length, unsigned char *key);
409409

410410
static void
411411
ReadReason(rfbClient* client)
@@ -585,7 +585,10 @@ HandleVncAuth(rfbClient *client)
585585
passwd[8] = '\0';
586586
}
587587

588-
rfbClientEncryptBytes(challenge, passwd);
588+
if (rfbClientEncryptBytes(challenge, passwd) != 0) {
589+
rfbClientLog("Encryption failed\n");
590+
return FALSE;
591+
}
589592

590593
/* Lose the password from memory */
591594
for (i = strlen(passwd); i >= 0; i--) {
@@ -733,8 +736,14 @@ HandleUltraMSLogonIIAuth(rfbClient *client)
733736
strncpy((char *)password, cred->userCredential.password, sizeof(password)-1);
734737
FreeUserCredential(cred);
735738

736-
rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key);
737-
rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key);
739+
if (rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key) != 0) {
740+
rfbClientLog("Encrypting username failed\n");
741+
return FALSE;
742+
}
743+
if (rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key) != 0) {
744+
rfbClientLog("Encrypting password failed\n");
745+
return FALSE;
746+
}
738747

739748
if (!WriteToRFBServer(client, (char *)pub, sizeof(pub))) return FALSE;
740749
if (!WriteToRFBServer(client, (char *)username, sizeof(username))) return FALSE;
@@ -789,8 +798,14 @@ HandleMSLogonAuth(rfbClient *client)
789798
pub = rfbClientSwap64IfLE(pub);
790799
key = rfbClientSwap64IfLE(key);
791800

792-
rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)&key);
793-
rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)&key);
801+
if (rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key) != 0) {
802+
rfbClientLog("Encrypting username failed\n");
803+
return FALSE;
804+
}
805+
if (rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key) != 0) {
806+
rfbClientLog("Encrypting password failed\n");
807+
return FALSE;
808+
}
794809

795810
if (!WriteToRFBServer(client, (char *)&pub, 8)) return FALSE;
796811
if (!WriteToRFBServer(client, (char *)username, sizeof(username))) return FALSE;

src/libvncserver/main.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -792,7 +792,11 @@ static rfbBool rfbDefaultPasswordCheck(rfbClientPtr cl,const char* response,int
792792
return(FALSE);
793793
}
794794

795-
rfbEncryptBytes(cl->authChallenge, passwd);
795+
if (rfbEncryptBytes(cl->authChallenge, passwd) != 0) {
796+
rfbErr("Encryption failed\n");
797+
free(passwd);
798+
return(FALSE);
799+
}
796800

797801
/* Lose the password from memory */
798802
for (i = strlen(passwd); i >= 0; i--) {
@@ -820,7 +824,10 @@ rfbBool rfbCheckPasswordByList(rfbClientPtr cl,const char* response,int len)
820824
for(passwds=(char**)cl->screen->authPasswdData;*passwds;passwds++,i++) {
821825
uint8_t auth_tmp[CHALLENGESIZE];
822826
memcpy((char *)auth_tmp, (char *)cl->authChallenge, CHALLENGESIZE);
823-
rfbEncryptBytes(auth_tmp, *passwds);
827+
if (rfbEncryptBytes(auth_tmp, *passwds) != 0) {
828+
rfbErr("Encryption failed\n");
829+
return(FALSE);
830+
}
824831

825832
if (memcmp(auth_tmp, response, len) == 0) {
826833
if(i>=cl->screen->authPasswdFirstViewOnly)

0 commit comments

Comments
 (0)