sourcedemos/activations/activations.xtl
1⍝!/usr/bin/env xetal
2⍝│ Activation functions, graphed: every function of this repository's
3⍝│ Activations library and its derivative over x from -6 to 6, two
4⍝│ charts a family (the functions, their slopes), drawn by
5⍝│ X_eTaL-libraries' Plot with an x axis of real values
6⍝│ (p:f_unctionChart!). The live page adds sliders for the parameters
7⍝│ and an editor for an activation of your own.
8
9"act:" u̲se< "Activations"
10"p:" u̲se< "Plot"
11
12⍝━━ Data
13
14⍝│ The x values: 241 from -6 to 6, a twentieth apart. The parameters:
15⍝│ leaky ReLU's slope, ELU's alpha, Swish's beta.
16xs ← 241 ᵖl̲inspace (-6.0, 6.0)
17leak ← 0.1
18alpha ← 1.0
19beta ← 1.5
20
21⍝│ A family's chart: the title, the axes' names and a name per curve,
22⍝│ then the curves, a row each over xs.
23ᵘc̲hart ← { labels ys → labels ᵖf̲unctionChart! '{ i → 2 241 r̲eshape xs c̲at i s̲elect ys } m̲ap r̲ange t̲ally ys }
24⍝│ A slope by central differences, for the gates, of which Activations
25⍝│ gives no derivative.
26ᵘs̲lope ← { f̲ x → 5000.0 × (f̲ x + 0.0001) − f̲ x − 0.0001 }
27⍝│ The gates as one-number layers, W = V = 1: x times its gate.
28one ← (1 1 r̲eshape 1.0, 1 1 r̲eshape 1.0)
29ᵘg̲lu1 ← { x → r̲avel one ᵃᶜᵗg̲lu 241 1 r̲eshape x }
30ᵘr̲eglu1 ← { x → r̲avel one ᵃᶜᵗr̲eglu 241 1 r̲eshape x }
31ᵘg̲eglu1 ← { x → r̲avel one ᵃᶜᵗg̲eglu 241 1 r̲eshape x }
32ᵘs̲wiglu1 ← { x → r̲avel one ᵃᶜᵗs̲wiglu 241 1 r̲eshape x }
33
34⍝━━ Classical
35
36⍝│ The squashing functions: sigmoid, tanh, hard sigmoid.
37classical ← 3 241 r̲eshape (ᵃᶜᵗs̲igmoid xs) c̲at (ᵃᶜᵗt̲anh xs) c̲at ᵃᶜᵗh̲ardSigmoid xs
38dClassical ← 3 241 r̲eshape (ᵃᶜᵗd̲sigmoid xs) c̲at (ᵃᶜᵗd̲tanh xs) c̲at ᵃᶜᵗd̲hardSigmoid xs
39_ ← ("Classical" "x" "f(x)" "sigmoid" "tanh" "hard sigmoid") ᵘc̲hart classical
40_ ← ("Slopes" "x" "f'(x)" "sigmoid" "tanh" "hard sigmoid") ᵘc̲hart dClassical
41
42⍝━━ Rectifiers
43
44⍝│ ReLU, leaky ReLU (its slope leak), ELU (alpha), SELU.
45rect ← 4 241 r̲eshape (ᵃᶜᵗr̲elu xs) c̲at (leak ᵃᶜᵗl̲eaky xs) c̲at (alpha ᵃᶜᵗe̲lu xs) c̲at ᵃᶜᵗs̲elu xs
46dRect ← 4 241 r̲eshape (ᵃᶜᵗd̲relu xs) c̲at (leak ᵃᶜᵗd̲leaky xs) c̲at (alpha ᵃᶜᵗd̲elu xs) c̲at ᵃᶜᵗd̲selu xs
47_ ← ("Rectifiers" "x" "f(x)" "ReLU" "leaky ReLU" "ELU" "SELU") ᵘc̲hart rect
48_ ← ("Slopes" "x" "f'(x)" "ReLU" "leaky ReLU" "ELU" "SELU") ᵘc̲hart dRect
49
50⍝━━ Smooth
51
52⍝│ Softplus, SiLU, Swish (beta), GELU (the tanh form and the exact
53⍝│ one), Mish, hard swish.
54smooth ← 7 241 r̲eshape (ᵃᶜᵗs̲oftplus xs) c̲at (ᵃᶜᵗs̲ilu xs) c̲at (beta ᵃᶜᵗs̲wish xs) c̲at (ᵃᶜᵗg̲elu xs) c̲at (ᵃᶜᵗg̲eluErf xs) c̲at (ᵃᶜᵗm̲ish xs) c̲at ᵃᶜᵗh̲ardSwish xs
55dSmooth ← 7 241 r̲eshape (ᵃᶜᵗd̲softplus xs) c̲at (ᵃᶜᵗd̲silu xs) c̲at (beta ᵃᶜᵗd̲swish xs) c̲at (ᵃᶜᵗd̲gelu xs) c̲at (ᵃᶜᵗd̲geluErf xs) c̲at (ᵃᶜᵗd̲mish xs) c̲at ᵃᶜᵗd̲hardSwish xs
56_ ← ("Smooth" "x" "f(x)" "softplus" "SiLU" "Swish" "GELU" "GELU (erf)" "Mish" "hard swish") ᵘc̲hart smooth
57_ ← ("Slopes" "x" "f'(x)" "softplus" "SiLU" "Swish" "GELU" "GELU (erf)" "Mish" "hard swish") ᵘc̲hart dSmooth
58
59⍝━━ Gated
60
61⍝│ The gated layers on one number, W = V = 1: x times sigmoid, ReLU,
62⍝│ GELU or SiLU of x; their slopes by central differences.
63gated ← 4 241 r̲eshape (ᵘg̲lu1 xs) c̲at (ᵘr̲eglu1 xs) c̲at (ᵘg̲eglu1 xs) c̲at ᵘs̲wiglu1 xs
64dGated ← 4 241 r̲eshape ('ᵘg̲lu1 ᵘs̲lope xs) c̲at ('ᵘr̲eglu1 ᵘs̲lope xs) c̲at ('ᵘg̲eglu1 ᵘs̲lope xs) c̲at 'ᵘs̲wiglu1 ᵘs̲lope xs
65_ ← ("Gated (W = V = 1)" "x" "f(x)" "GLU" "ReGLU" "GEGLU" "SwiGLU") ᵘc̲hart gated
66_ ← ("Slopes (by differences)" "x" "f'(x)" "GLU" "ReGLU" "GEGLU" "SwiGLU") ᵘc̲hart dGated
67
68⍝━━ The ranges
69
70⍝│ Each family's least and greatest values over -6..6, a row a family
71⍝│ (classical, rectifiers, smooth, gated); then its slopes'.
72ᵘs̲pan ← { f → ('m̲in r̲/ r̲avel f) c̲at 'm̲ax r̲/ r̲avel f }
734 2 r̲eshape (ᵘs̲pan classical) c̲at (ᵘs̲pan rect) c̲at (ᵘs̲pan smooth) c̲at ᵘs̲pan gated
744 2 r̲eshape (ᵘs̲pan dClassical) c̲at (ᵘs̲pan dRect) c̲at (ᵘs̲pan dSmooth) c̲at ᵘs̲pan dGated