vmstat: precision issues in unitConvert()
Fix conversion errors due to precision issues in function unitConvert For example: unitConvert(98720620) will return 98720624, not 98720620. Because we do (unsigned long)(float)98720620 in function unitConvert and this is wrong! We should do (unsigned long)(double)98720620 here. Signed-off-by: Craig Small <csmall@dropbear.xyz> References: procps-ng/procps!75
This commit is contained in:
1 parent
68e2c0fc53
commit
0a5ccbdd42
2 files changed
+3
-2
No files matched your search
+2
-2
@@ -336,8 +336,8 @@ static void new_header(void)
|
||||
|
||||
static unsigned long unitConvert(unsigned long size)
|
||||
{
|
||||
float cvSize;
|
||||
cvSize = (float)size / dataUnit * ((statMode == SLABSTAT) ? 1 : 1024);
|
||||
double cvSize;
|
||||
cvSize = (double)size / dataUnit * ((statMode == SLABSTAT) ? 1 : 1024);
|
||||
return ((unsigned long)cvSize);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user