programlibs/Integrals/demos/gelu.xtl

source · imports integ: libs/Integrals/src/Integrals.xtlm; act: libs/Activations/src/Activations.xtl

ᵘp̲hi : Float -> Float

function · line 13
ᵘp̲hi ← { u → (e̲xp n̲eg (u × u) ÷ 2.0) ÷ (2.0 × p̲i @) ^ 0.5 }
Used in: ᵘg̲elu

ᵘg̲elu : Float -> Float

function · line 15
ᵘg̲elu ← { x → x × 0.5 + @ ⁱⁿᵗᵉᵍi̲ntegral< "integral u:=0.0..x by simpson 32 | u:p_hi u" }
ⁱⁿᵗᵉᵍi̲ntegral< expands to
({ @ →
  g1:a ← 0.0
  g2:b ← x
  g3:h ← (g2:b − g1:a) ÷ 32.0
  u ← g1:a + g3:h × f̲loat o̲ffsets 33
  g4:w ← (1.0 c̲at (31 r̲eshape 4.0 2.0) c̲at 1.0) ÷ 3.0
  g3:h × '+ r̲/ g4:w × ᵘp̲hi u
} @)

ᵘa̲pprox : Float -> Float

function · line 17
ᵘa̲pprox ← { x → 0.5 × x × 1.0 + ᵃᶜᵗt̲anh ((2.0 ÷ p̲i @) ^ 0.5) × x + 0.044715 × x ^ 3 }

xs : Float

value · line 18
xs ← -3.0 -2.0 -1.0 -0.5 0.0 0.5 1.0 2.0 3.0

g : Float

value · line 19
g ← 'ᵘg̲elu e̲ach xs

fine : Float

value · line 23
fine ← -4.0 + 0.05 × f̲loat o̲ffsets 161

ᵘn̲ormal : (Float, Float) -> Float -> Float

function · line 30
ᵘn̲ormal ← { (m, s) x → (e̲xp n̲eg ((x − m) ^ 2) ÷ 2.0 × s × s) ÷ s × (2.0 × p̲i @) ^ 0.5 }
Used in: kl

kl : Float

value · line 31
kl ← @ ⁱⁿᵗᵉᵍi̲ntegral< "integral x:=-10.0..10.0 by simpson 400 | ((0.0, 1.0) u:n_ormal x) * l_og ((0.0, 1.0) u:n_ormal x) / (1.0, 2.0) u:n_ormal x"
ⁱⁿᵗᵉᵍi̲ntegral< expands to
({ @ →
  g5:a ← -10.0
  g6:b ← 10.0
  g7:h ← (g6:b − g5:a) ÷ 400.0
  x ← g5:a + g7:h × f̲loat o̲ffsets 401
  g8:w ← (1.0 c̲at (399 r̲eshape 4.0 2.0) c̲at 1.0) ÷ 3.0
  g7:h × '+ r̲/ g8:w × ((0.0, 1.0) ᵘn̲ormal x) × l̲og ((0.0, 1.0) ᵘn̲ormal x) ÷ (1.0, 2.0) ᵘn̲ormal x
} @)

closed : Float

value · line 32
closed ← (l̲og 2.0) + ((1.0 + 1.0) ÷ 2.0 × 4.0) − 0.5