mir/libreoffice
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

recognize NaN with no bits set in lower word as error

Browse Source 
Which can happen for example for -nan(0x8000000000000) as a result of
calculating with -inf. This was displayed as NaN instead of a proper error
value, now #NUM!

Example test case:
=FORECAST.ETS.ADD(50, {-1,-2,-3,-4}, {10,20,30,40})

Change-Id: I1e1d95e1f188e0036b72be37dd20039c9a9a13f6
This commit is contained in:
Eike Rathke
2016-09-13 15:26:09 +02:00
parent 7aa10d2be1
commit 4ef10fce39
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
include/formula/errorcodes.hxx
View File

@@ -105,11 +105,14 @@ inline sal_uInt16 GetDoubleErrorValue( double fVal )
return 0; return 0;
if ( ::rtl::math::isInf( fVal ) ) if ( ::rtl::math::isInf( fVal ) )
return errIllegalFPOperation; // normal INF return errIllegalFPOperation; // normal INF
sal_uInt32 nErr = reinterpret_cast< sal_math_Double * >( sal_uInt32 nErr = reinterpret_cast< sal_math_Double * >( &fVal)->nan_parts.fraction_lo;
&fVal)->nan_parts.fraction_lo;
if ( nErr & 0xffff0000 ) if ( nErr & 0xffff0000 )
return errNoValue; // just a normal NAN return errNoValue; // just a normal NAN
return (sal_uInt16)(nErr & 0x0000ffff); // any other error if (!nErr)
// Another NAN, e.g. -nan(0x8000000000000) from calculating with -inf
return errIllegalFPOperation;
// Any other error known to us as error code.
return (sal_uInt16)(nErr & 0x0000ffff);
} }
} // namespace formula } // namespace formula