setmode bug.

[Kyle Butt <kylebutt@xxxxxxxxx>]

Attached is a patch that fixes a few things in setmode as was
discussed about a recent fix to uudecode.

-set errno to EINVAL when we have an invalid file mode.
-make sure errno is set for a malloc error
-fix a lingering bug that missed the first character in an octal mode
        so that 844 was not being recognized as an invalid mode.
        also affected chmod.

-update uudecode to match the changes.

compiles in make world and I tested them to make sure they work.


Index: uudecode.c
===================================================================
RCS file: /home/ncvs/src/usr.bin/uudecode/uudecode.c,v
retrieving revision 1.3
diff -u -r1.3 uudecode.c
--- uudecode.c	22 Dec 2004 11:25:51 -0000	1.3
+++ uudecode.c	23 Dec 2004 07:41:15 -0000
@@ -207,9 +207,9 @@
 
 	errno = 0;
 	if ((handle = setmode(p)) == NULL) {
-		if (!errno)
+		if (errno == EINVAL)
 			warnx("invalid file mode: %s", infile);
-		else
+		else 
 			warn("setmode malloc failed: %s", infile);
 
 		return(1);
Index: libc/gen/setmode.c
===================================================================
RCS file: /home/ncvs/src/lib/libc/gen/setmode.c,v
retrieving revision 1.4
diff -u -r1.4 setmode.c
--- libc/gen/setmode.c	6 Jun 2004 15:05:55 -0000	1.4
+++ libc/gen/setmode.c	23 Dec 2004 08:56:50 -0000
@@ -45,6 +45,7 @@
 #include <signal.h>
 #include <stddef.h>
 #include <stdlib.h>
+#include <errno.h>
 
 #ifdef SETMODE_DEBUG
 #include <stdio.h>
@@ -151,9 +152,13 @@
 	if (set >= endset) {						\
 		BITCMD *newset;				\
 		setlen += SET_LEN_INCR;					\
+		errno = 0;                                            \
 		newset = realloc(saveset, sizeof(BITCMD) * setlen);	\
-		if (!saveset)						\
+		if (!saveset) {													\
+			 if (errno == 0)                                   \
+				  errno = ENOMEM;                               \
 			return (NULL);					\
+		}                                         \
 		set = newset + (set - saveset);				\
 		saveset = newset;					\
 		endset = newset + (setlen - 2);				\
@@ -173,8 +178,11 @@
 	mode_t mask;
 	int equalopdone=0, permXbits, setlen;
 
-	if (!*p)
+	if (p == NULL || p[0] == '\0') 
+	{
+		errno = EINVAL;
 		return (NULL);
+	}
 
 	/*
 	 * Get a copy of the mask for the permissions that are mask relative.
@@ -190,8 +198,15 @@
 
 	setlen = SET_LEN + 2;
 
+	errno = 0;
 	if ((set = malloc((u_int)(sizeof(BITCMD) * setlen))) == NULL)
-		return (NULL);
+	{
+		 if(errno == 0)
+		 {
+			  errno = ENOMEM;
+		 }
+		 return (NULL);
+	}
 	saveset = set;
 	endset = set + (setlen - 2);
 
@@ -203,13 +218,17 @@
 		perm = (mode_t)strtol(p, NULL, 8);
 		if (perm & ~(STANDARD_BITS|S_ISTXT)) {
 			free(saveset);
+			errno = EINVAL;
 			return (NULL);
 		}
-		while (*++p)
+		while (*p != '\0') {
 			if (*p < '0' || *p > '7') {
 				free(saveset);
+				errno = EINVAL;
 				return (NULL);
 			}
+			p++;
+		}
 		ADDCMD('=', (STANDARD_BITS|S_ISTXT), perm, mask);
 		return (saveset);
 	}
@@ -239,8 +258,9 @@
 			}
 		}
 
-getop:		if ((op = *p++) != '+' && op != '-' && op != '=') {
+getop:	if ((op = *p++) != '+' && op != '-' && op != '=') {
 			free(saveset);
+			errno = EINVAL;
 			return (NULL);
 		}
 		if (op == '=')
Index: libcr/gen/setmode.c
===================================================================
RCS file: /home/ncvs/src/lib/libcr/gen/setmode.c,v
retrieving revision 1.4
diff -u -r1.4 setmode.c
--- libcr/gen/setmode.c	5 Jul 2004 17:30:59 -0000	1.4
+++ libcr/gen/setmode.c	23 Dec 2004 08:57:42 -0000
@@ -45,6 +45,7 @@
 #include <signal.h>
 #include <stddef.h>
 #include <stdlib.h>
+#include <errno.h>
 
 #ifdef SETMODE_DEBUG
 #include <stdio.h>
@@ -151,9 +152,13 @@
 	if (set >= endset) {						\
 		BITCMD *newset;				\
 		setlen += SET_LEN_INCR;					\
+		errno = 0;                          \
 		newset = realloc(saveset, sizeof(BITCMD) * setlen);	\
-		if (!saveset)						\
-			return (NULL);					\
+		if (!saveset) {													\
+			 if (errno == 0)                                   \
+				  errno = ENOMEM;                               \
+			 return (NULL);												\
+		}                                                     \
 		set = newset + (set - saveset);				\
 		saveset = newset;					\
 		endset = newset + (setlen - 2);				\
@@ -173,8 +178,11 @@
 	mode_t mask;
 	int equalopdone=0, permXbits, setlen;
 
-	if (!*p)
-		return (NULL);
+	if (p == NULL || p[0] == NULL)
+	{
+		 errno = EINVAL;
+		 return (NULL);
+	}
 
 	/*
 	 * Get a copy of the mask for the permissions that are mask relative.
@@ -190,8 +198,16 @@
 
 	setlen = SET_LEN + 2;
 
+	errno = 0;
 	if ((set = malloc((u_int)(sizeof(BITCMD) * setlen))) == NULL)
-		return (NULL);
+	{
+		 if(errno == 0)
+		 {
+			  /* If it wasn't set, set to a meaningful value. */
+			  errno = ENOMEM;
+		 }
+		 return (NULL);
+	}
 	saveset = set;
 	endset = set + (setlen - 2);
 
@@ -203,13 +219,17 @@
 		perm = (mode_t)strtol(p, NULL, 8);
 		if (perm & ~(STANDARD_BITS|S_ISTXT)) {
 			free(saveset);
+			errno = EINVAL;
 			return (NULL);
 		}
-		while (*++p)
+		while (*p != '\0') {
 			if (*p < '0' || *p > '7') {
 				free(saveset);
+				errno = EINVAL;
 				return (NULL);
 			}
+			p++;
+		}
 		ADDCMD('=', (STANDARD_BITS|S_ISTXT), perm, mask);
 		return (saveset);
 	}
@@ -241,6 +261,7 @@
 
 getop:		if ((op = *p++) != '+' && op != '-' && op != '=') {
 			free(saveset);
+			errno = EINVAL;
 			return (NULL);
 		}
 		if (op == '=')