train-xor.xtl

1⍝!/usr/bin/env xetal
2⍝# XOR learned, the network and its training each written as one line:
3⍝# t_rain< writes the training step (backpropagation and Adam) from the
4⍝# same spec n_etwork< reads. Small random weights at the start.
5⍝# Run with "just demo-lib Net train-xor"; see what was written with
6⍝# "just expand-lib Net train-xor".
7
8ⁿⁿ⁼u̲se< "NN"
9ⁿᵉᵗ⁼u̲se< "Net"
10X ← 4 2 r̲eshape 0.0 0.0 0.0 1.0 1.0 0.0 1.0 1.0
11y ← 1 2 2 1
12Y ← 2 ⁿⁿo̲neHot y
13lr ← 0.05
14"u:s_tep X Y lr" ⁿᵉᵗt̲rain< "2 4 tanh 2 softmax"
ⁿᵉᵗt̲rain< expands toᵘs̲tep ← { (W1, W2, M1, M2, V1, V2, k) →
  A0 ← X
  A1 ← ⁿⁿt̲anh A0 ⁿⁿd̲ense W1
  A2 ← ⁿⁿs̲oftmax A1 ⁿⁿd̲ense W2
  D2 ← (A2 − Y) ÷ f̲loat t̲ally X
  D1 ← (D2 '+ '× i̲nner o̲\ -1 d̲rop W2) × 1.0 − A1 × A1
  G1 ← (o̲\ A0 c̲at₂ ((t̲ally A0) c̲at 1) r̲eshape 1.0) '+ '× i̲nner D1
  G2 ← (o̲\ A1 c̲at₂ ((t̲ally A1) c̲at 1) r̲eshape 1.0) '+ '× i̲nner D2
  k ← 1.0 + k
  M1 ← (0.9 × M1) + 0.1 × G1
  V1 ← (0.999 × V1) + 0.001 × G1 × G1
  W1 ← W1 − lr × (M1 ÷ 1.0 − 0.9 ^ k) ÷ 0.00000001 + (V1 ÷ 1.0 − 0.999 ^ k) ^ 0.5
  M2 ← (0.9 × M2) + 0.1 × G2
  V2 ← (0.999 × V2) + 0.001 × G2 × G2
  W2 ← W2 − lr × (M2 ÷ 1.0 − 0.9 ^ k) ÷ 0.00000001 + (V2 ÷ 1.0 − 0.999 ^ k) ^ 0.5
  (W1, W2, M1, M2, V1, V2, k)
}
15w1 ← (f̲loat (3 4 r̲eshape r̲oll! 12 r̲eshape 201) − 101) ÷ 100.0
16w2 ← (f̲loat (5 2 r̲eshape r̲oll! 10 r̲eshape 201) − 101) ÷ 100.0
17⍝# The state after 300 steps, a tuple: the weights first.
18(v1, v2, _, _, _, _, _) ← 300 'ᵘs̲tep p̲ower @ ⁿᵉᵗs̲tate< "w1 w2"
ⁿᵉᵗs̲tate< expands to((w1, w2, 0.0 × w1, 0.0 × w2, 0.0 × w1, 0.0 × w2, 0.0))
19ᵘb̲efore ← "2 4 tanh 2 softmax" ⁿᵉᵗn̲etwork< "w1 w2"
ⁿᵉᵗn̲etwork< expands to({ x → ⁿⁿs̲oftmax (ⁿⁿt̲anh x ⁿⁿd̲ense w1) ⁿⁿd̲ense w2 })
20ᵘa̲fter ← "2 4 tanh 2 softmax" ⁿᵉᵗn̲etwork< "v1 v2"
ⁿᵉᵗn̲etwork< expands to({ x → ⁿⁿs̲oftmax (ⁿⁿt̲anh x ⁿⁿd̲ense v1) ⁿⁿd̲ense v2 })
21⍝ The loss before and after 300 steps, and the classes read after.
22(Y ⁿⁿc̲rossEntropy ᵘb̲efore X) c̲at Y ⁿⁿc̲rossEntropy ᵘa̲fter X
23ⁿⁿa̲rgmax ᵘa̲fter X
24y ⁿⁿa̲ccuracy ᵘa̲fter X