summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorRich Felker <dalias@aerifal.cx>2014-03-09 03:09:49 -0400
committerRich Felker <dalias@aerifal.cx>2014-03-09 03:09:49 -0400
commit9743a399bf4d6da9a1dbdf7e8df07284c97df16f (patch)
treec9477850f2abba3b76123e1a2299b379199be9c4 /src
parentba231cf9e5923b6216081e9a626465c6643ce4d3 (diff)
downloadmusl-9743a399bf4d6da9a1dbdf7e8df07284c97df16f.tar.gz
fix incorrect rounding in printf floating point corner cases
the printf floating point formatting code contains an optimization to avoid computing digits that will be thrown away by rounding at the specified (or default) precision. while it was correctly retaining all places up to the last decimal place to be printed, it was not retaining enough precision to see the next nonzero decimal place in all cases. this could cause incorrect rounding down in round-to-even (default) rounding mode, for example, when printing 0.5+DBL_EPSILON with "%.0f". in the fix, LDBL_MANT_DIG/3 is a lazy (non-sharp) upper bound on the number of zeros between any two nonzero decimal digits.
Diffstat (limited to 'src')
-rw-r--r--src/stdio/vfprintf.c4
1 files changed, 2 insertions, 2 deletions
diff --git a/src/stdio/vfprintf.c b/src/stdio/vfprintf.c
index 85701726..31c3d5dd 100644
--- a/src/stdio/vfprintf.c
+++ b/src/stdio/vfprintf.c
@@ -314,7 +314,7 @@ static int fmt_fp(FILE *f, long double y, int w, int p, int fl, int t)
}
while (e2<0) {
uint32_t carry=0, *b;
- int sh=MIN(9,-e2);
+ int sh=MIN(9,-e2), need=1+(p+LDBL_MANT_DIG/3+8)/9;
for (d=a; d<z; d++) {
uint32_t rm = *d & (1<<sh)-1;
*d = (*d>>sh) + carry;
@@ -324,7 +324,7 @@ static int fmt_fp(FILE *f, long double y, int w, int p, int fl, int t)
if (carry) *z++ = carry;
/* Avoid (slow!) computation past requested precision */
b = (t|32)=='f' ? r : a;
- if (z-b > 2+p/9) z = b+2+p/9;
+ if (z-b > need) z = b+need;
e2+=sh;
}