make LIBC sqrtf signal EDOM/FE_INVALID

Author: ZERICO2005

src/crt/fpsqrt.src

@@ -18,7 +18,7 @@ __fpsqrt: ; CHECK: same(bitcast(float, pair8_24_t, { out.BC, out.A }), (float)sq
 	sbc	hl, bc
 	jr	z, .L.zero
 	rlca
-	jr	c, .L.nan
+	jr	c, .L.return_nan
 	inc	e
 	jr	z, .L.nonfinite
 	push	hl
@@ -107,7 +107,9 @@ __fpsqrt: ; CHECK: same(bitcast(float, pair8_24_t, { out.BC, out.A }), (float)sq
 	pop	de
 	ret
 
-.L.nan:
+.L.return_nan:
+	; input was negative and not -0.0
+	; return negative NaN
 	sbc	a, a
 	set	7, b
 	pop	hl

src/libc/sqrtf.src

@@ -19,13 +19,25 @@ _sqrtf:
 	pop	bc
 	ex	(sp), hl
 	ld	a, l
+	; assumes [-infinity, -FLT_TRUE_MIN] returns -NaN
 	call	__fpsqrt
 	push	bc
 	ex	(sp), hl
 	push	de
 	ld	e, a
+	inc	a		; test if the result was -NaN
+	; +NaN returns here while -NaN does not, but this is not worth fixing.
+	ret	nz
+	push	hl
+	ld	hl, 4		; EDOM
+	ld	(_errno), hl
+	ld	hl, ___fe_cur_env
+	set	4, (hl)		; FE_INVALID
+	pop	hl
 	ret
 
 	.extern	__fpsqrt
+	.extern	__errno
+	.extern	___fe_cur_env
 
 .endif