Net.xtlm

1⍝# Net: a network written as one line -- its layer sizes and activations become ordinary NN calls
2⍝#
3⍝# Import it with an alias of your choice: "net:" u_se< "Net".
4⍝# Put libs/Net/src on XETAL_PATH ("just path"); the reference is libs/Net/docs.
5⍝# Names with m: are macros; the others are private to this file.
6⍝#
7⍝# A macro is a function from the source text written left and right of
8⍝# its call to the source that replaces the call, before the program is
9⍝# compiled (X_eTaL MC10); xetal expand FILE shows the result. A spec
10⍝# is a list of words: sizes and activations, the input's size first:
11⍝# "784 128 relu 10 softmax". A size after the first is a dense layer
12⍝# (nn:d_ense, with its weights: one array per layer, the bias its last
13⍝# row); an activation is a function of NN (relu, sigmoid, tanh,
14⍝# softmax, logSoftmax). The expansion calls NN under the alias nn:
15⍝# (an expansion cannot ask which alias the caller chose), so import
16⍝# NN as "nn:" u_se< "NN".
17
18⍝## Reading a spec (private)
19
20⍝# The words of t: runs of characters other than white space, boxed
21⍝# (after X_eTaL's Combinators.xtlm).
22ʰw̲ords ← { t → (n̲ot t m̲ember? " \n\t") p̲artition t }
23⍝# The boxed texts b joined into one text ("" for none).
24ʰj̲oin ← { b →
25  0 = t̲ally b ? ""
26  d̲isclose '{ x y → e̲nclose (d̲isclose x) c̲at d̲isclose y } r̲/ b
27}
28⍝# Whether the text w is a whole number (a layer's size).
29ʰs̲ize? ← { w → '∧ r̲/ w m̲ember? "0123456789" }
30⍝# Whether the text w names an activation.
31ʰa̲ct? ← { w → '∨ r̲/ '{ a → w m̲atch d̲isclose a } e̲ach ʰw̲ords "relu sigmoid tanh softmax logSoftmax" }
32⍝# An activation's function in NN: relu -> nn:r_elu.
33ʰa̲ct ← { w → "nn:" c̲at (1 t̲ake w) c̲at "_" c̲at 1 d̲rop w }
34⍝# The spec's sizes, as Ints.
35ʰs̲izes ← { ws → f̲loor n̲umbers ʰj̲oin '{ w → (ʰs̲ize? d̲isclose w) ? (d̲isclose w) c̲at " "◆ "" } m̲ap ws }
36⍝# What a spec looks like, for the messages.
37specShape ← "a spec is sizes and activations, the input's size first: \"784 128 relu 10 softmax\""
38⍝# What is wrong with the spec's words ws, or "".
39ʰw̲rong ← { ws →
40  2 > t̲ally ws ? specShape
41  n̲ot ʰs̲ize? d̲isclose f̲irst ws ? specShape
42  bad ← '{ w → n̲ot (ʰs̲ize? d̲isclose w) ∨ ʰa̲ct? d̲isclose w } e̲ach ws
43  0 < '+ r̲/ 0 + bad ? "not a size or an activation (relu, sigmoid, tanh, softmax, logSoftmax): " c̲at d̲isclose f̲irst (w̲here bad) s̲elect ws
44  ""
45}
46
47⍝## Writing the forward pass (private)
48
49⍝# The forward function for the spec's words ws and the weight names
50⍝# ns: a lambda of nn: calls, the input x through each layer in turn.
51ʰf̲unction ← { ws ns →
52  size ← '{ w → ʰs̲ize? d̲isclose w } e̲ach ws
53  k ← '+ s̲\ 0 + size
54  pre ← '{ i →
55    n̲ot i s̲elect size ? (ʰa̲ct d̲isclose i s̲elect ws) c̲at " "
56    2 = i s̲elect k ? ""
57    "("
58  } m̲ap 1 d̲rop r̲ange t̲ally ws
59  post ← '{ i →
60    n̲ot i s̲elect size ? ""
61    d ← " nn:d_ense " c̲at d̲isclose ((i s̲elect k) − 1) s̲elect ns
62    2 = i s̲elect k ? d
63    ")" c̲at d
64  } m̲ap 1 d̲rop r̲ange t̲ally ws
65  "{ x -> " c̲at (ʰj̲oin r̲ev pre) c̲at "x" c̲at (ʰj̲oin post) c̲at " }"
66}
67⍝# What is wrong with the weight names ns for the spec's words ws, or "".
68ʰc̲ount ← { ws ns →
69  dense ← -1 + '+ r̲/ '{ w → 0 + ʰs̲ize? d̲isclose w } e̲ach ws
70  (t̲ally ns) = dense ? ""
71  "name one weight array per dense layer: " c̲at (f̲ormat dense) c̲at " for this spec, " c̲at (f̲ormat t̲ally ns) c̲at " given"
72}
73
74⍝## Networks
75
76⍝# A forward function for the network the spec on the left describes,
77⍝# using the weight arrays named on the right, one per dense layer in
78⍝# order: an ordinary lambda of nn: calls, a batch (one example per
79⍝# row) in, the last layer's output out.
80ᵐn̲etwork< ← { spec weights →
81  ws ← ʰw̲ords spec
82  ns ← ʰw̲ords weights
83  0 < t̲ally ʰw̲rong ws ? "bad-macro-argument left" ⎕R̲EJECT ʰw̲rong ws
84  0 < t̲ally ws ʰc̲ount ns ? "bad-macro-argument right" ⎕R̲EJECT ws ʰc̲ount ns
85  ws ʰf̲unction ns
86}
87
88⍝# A whole model from one spec: on the left the function's name and a
89⍝# prefix ("u:d_eep c"), on the right the spec. It writes, as
90⍝# statements, each dense layer's weights read from data/<prefix>K.txt
91⍝# and reshaped as the spec says (inputs + 1 by outputs; the program
92⍝# stops with an error naming the layer when a file does not hold that
93⍝# many numbers, since r_eshape alone would silently repeat or cut the
94⍝# data), then the function, as n_etwork< writes it. The spec is the
95⍝# one place the network is described.
96ᵐm̲odel< ← { what spec →
97  ws ← ʰw̲ords spec
98  0 < t̲ally ʰw̲rong ws ? "bad-macro-argument right" ⎕R̲EJECT ʰw̲rong ws
99  nm ← ʰw̲ords what
100  2 ≠ t̲ally nm ? "bad-macro-argument left" ⎕R̲EJECT "write the function's name and the weight files' prefix: \"u:d_eep c\" (for data/c1.txt, data/c2.txt, ...)"
101  name ← d̲isclose f̲irst nm
102  prefix ← d̲isclose 2 s̲elect nm
103  s ← ʰs̲izes ws
104  ns ← '{ i → prefix c̲at f̲ormat i } m̲ap r̲ange -1 + t̲ally s
105  load ← '{ i →
106    w ← d̲isclose i s̲elect ns
107    need ← (1 + i s̲elect s) × (i + 1) s̲elect s
108    w c̲at " := { v -> " c̲at (f̲ormat need) c̲at " = t_ally v ? " c̲at (f̲ormat 1 + i s̲elect s) c̲at " " c̲at (f̲ormat (i + 1) s̲elect s) c̲at " r_eshape v; []P_ANIC \"data/" c̲at w c̲at ".txt holds \" c_at (f_ormat t_ally v) c_at \" numbers; layer " c̲at (f̲ormat i) c̲at " of \\\"" c̲at spec c̲at "\\\" (" c̲at (f̲ormat i s̲elect s) c̲at " inputs and a bias, " c̲at (f̲ormat (i + 1) s̲elect s) c̲at " outputs) needs " c̲at (f̲ormat need) c̲at "\" } n_umbers []N_GET \"data/" c̲at w c̲at ".txt\"\n"
109  } m̲ap r̲ange -1 + t̲ally s
110  (ʰj̲oin load) c̲at name c̲at " := " c̲at ws ʰf̲unction ns
111}
112
113⍝## Writing backpropagation (private)
114
115⍝# The function of NN for an activation word (relu -> nn:r_elu); "" for none.
116ʰa̲ctFn ← { w → 0 = t̲ally w ? ""◆ (ʰa̲ct w) c̲at " " }
117⍝# Its slope at its output A (the name of the array), times which the
118⍝# error goes back through it; "" when there is none.
119ʰs̲lope ← { w a →
120  w m̲atch "relu" ? " * f_loat " c̲at a c̲at " > 0.0"
121  w m̲atch "tanh" ? " * 1.0 - " c̲at a c̲at " * " c̲at a
122  w m̲atch "sigmoid" ? " * " c̲at a c̲at " * 1.0 - " c̲at a
123  ""
124}
125⍝# The activation after each dense layer of the spec's words ws (a box
126⍝# per layer, "" for none).
127ʰa̲cts ← { ws →
128  size ← '{ w → ʰs̲ize? d̲isclose w } e̲ach ws
129  at ← 1 d̲rop w̲here size
130  '{ p → (p < t̲ally ws) ∧ (n̲ot (p + 1) s̲elect size) ? d̲isclose (p + 1) s̲elect ws◆ "" } m̲ap at
131}
132⍝# N as text.
133ʰt̲xt ← { n → f̲ormat n }
134
135⍝# The names P1 .. PL as X_eTaL's tuple text, "(P1, P2)"; one alone is
136⍝# not a tuple, "P1" (X_eTaL has no one-element tuple).
137ʰt̲uple ← { p L →
138  L = 1 ? p c̲at "1"
139  "(" c̲at (ʰj̲oin '{ l → (l = 1) ? p c̲at ʰt̲xt l◆ ", " c̲at p c̲at ʰt̲xt l } m̲ap r̲ange L) c̲at ")"
140}
141
142⍝# The lines that run the spec's network forward from the inputs
143⍝# (the weights W1 .. WL bound by the caller) and back from the targets,
144⍝# and bind each layer's gradient G1 .. GL. ws: the spec's words; xy:
145⍝# the inputs' and targets' names, boxed.
146ʰb̲ackprop ← { ws xy →
147  x ← d̲isclose 1 s̲elect xy
148  y ← d̲isclose 2 s̲elect xy
149  acts ← ʰa̲cts ws
150  L ← t̲ally acts
151  r ← r̲ange L
152  fwd ← ʰj̲oin '{ l → "  A" c̲at (ʰt̲xt l) c̲at " := " c̲at (ʰa̲ctFn d̲isclose l s̲elect acts) c̲at "A" c̲at (ʰt̲xt l − 1) c̲at " nn:d_ense W" c̲at (ʰt̲xt l) c̲at "\n" } m̲ap r
153  dL ← "  D" c̲at (ʰt̲xt L) c̲at " := (A" c̲at (ʰt̲xt L) c̲at " - " c̲at y c̲at ") / f_loat t_ally " c̲at x c̲at "\n"
154  back ← ʰj̲oin '{ l → "  D" c̲at (ʰt̲xt l) c̲at " := (D" c̲at (ʰt̲xt l + 1) c̲at " '+ '* i_nner o_\\ -1 d_rop W" c̲at (ʰt̲xt l + 1) c̲at ")" c̲at ((d̲isclose l s̲elect acts) ʰs̲lope "A" c̲at ʰt̲xt l) c̲at "\n" } m̲ap r̲ev -1 d̲rop r
155  grads ← ʰj̲oin '{ l → "  G" c̲at (ʰt̲xt l) c̲at " := (o_\\ A" c̲at (ʰt̲xt l − 1) c̲at " c_at_2 ((t_ally A" c̲at (ʰt̲xt l − 1) c̲at ") c_at 1) r_eshape 1.0) '+ '* i_nner D" c̲at (ʰt̲xt l) c̲at "\n" } m̲ap r
156  "  A0 := " c̲at x c̲at "\n" c̲at fwd c̲at dL c̲at back c̲at grads
157}
158⍝# What is wrong with a trainable spec's words ws, or "".
159ʰt̲rainable ← { ws →
160  0 < t̲ally ʰw̲rong ws ? ʰw̲rong ws
161  n̲ot (d̲isclose f̲irst r̲ev ws) m̲atch "softmax" ? "a network trained with cross-entropy ends in softmax"
162  ""
163}
164
165⍝## Training
166
167⍝# The gradient of the cross-entropy loss by each layer's weights, for
168⍝# the network the spec on the right describes: on the left the
169⍝# function's name, the inputs' and the targets' (one-hot rows):
170⍝# "u:g_rad X Y". The function takes the weights as a tuple
171⍝# (W1, .., WL) and gives the gradients as a tuple (G1, .., GL),
172⍝# each the shape of its layer's weights: backpropagation, written out.
173ᵐg̲radient< ← { what spec →
174  ws ← ʰw̲ords spec
175  0 < t̲ally ʰt̲rainable ws ? "bad-macro-argument right" ⎕R̲EJECT ʰt̲rainable ws
176  nm ← ʰw̲ords what
177  3 ≠ t̲ally nm ? "bad-macro-argument left" ⎕R̲EJECT "write the function's name, then the inputs' and the targets': \"u:g_rad X Y\""
178  L ← t̲ally ʰa̲cts ws
179  (d̲isclose 1 s̲elect nm) c̲at " := { " c̲at ("W" ʰt̲uple L) c̲at " ->\n" c̲at (ws ʰb̲ackprop 1 d̲rop nm) c̲at "  " c̲at ("G" ʰt̲uple L) c̲at "\n}"
180}
181
182⍝# A training step for the network the spec on the right describes:
183⍝# on the left the function's name, the inputs', the targets' (one-hot
184⍝# rows) and the learning rate's ("u:s_tep X Y lr"). The step takes the
185⍝# state, a tuple of 3L + 1 arrays (each layer's weights, Adam's running
186⍝# averages of gradients and of their squares, the step count; @
187⍝# net:s_tate< writes the first), and gives the next: the forward pass,
188⍝# each layer's gradient by backpropagation (as g_radient< writes it),
189⍝# and Adam. The spec must end in softmax.
190ᵐt̲rain< ← { what spec →
191  ws ← ʰw̲ords spec
192  0 < t̲ally ʰt̲rainable ws ? "bad-macro-argument right" ⎕R̲EJECT ʰt̲rainable ws
193  nm ← ʰw̲ords what
194  4 ≠ t̲ally nm ? "bad-macro-argument left" ⎕R̲EJECT "write the step's name, then the inputs', the targets' and the learning rate's: \"u:s_tep X Y lr\""
195  lr ← d̲isclose 4 s̲elect nm
196  L ← t̲ally ʰa̲cts ws
197  r ← r̲ange L
198  adam ← ʰj̲oin '{ l →
199    i ← ʰt̲xt l
200    "  M" c̲at i c̲at " := (0.9 * M" c̲at i c̲at ") + 0.1 * G" c̲at i c̲at "\n  V" c̲at i c̲at " := (0.999 * V" c̲at i c̲at ") + 0.001 * G" c̲at i c̲at " * G" c̲at i c̲at "\n  W" c̲at i c̲at " := W" c̲at i c̲at " - " c̲at lr c̲at " * (M" c̲at i c̲at " / 1.0 - 0.9 ^ k) / 0.00000001 + (V" c̲at i c̲at " / 1.0 - 0.999 ^ k) ^ 0.5\n"
201  } m̲ap r
202  names ← ʰj̲oin '{ l → "W" c̲at (ʰt̲xt l) c̲at ", " } m̲ap r
203  names ← names c̲at (ʰj̲oin '{ l → "M" c̲at (ʰt̲xt l) c̲at ", " } m̲ap r) c̲at ʰj̲oin '{ l → "V" c̲at (ʰt̲xt l) c̲at ", " } m̲ap r
204  state ← "(" c̲at names c̲at "k)"
205  (d̲isclose 1 s̲elect nm) c̲at " := { " c̲at state c̲at " ->\n" c̲at (ws ʰb̲ackprop 1 d̲rop 3 t̲ake nm) c̲at "  k := 1.0 + k\n" c̲at adam c̲at "  " c̲at state c̲at "\n}"
206}
207
208⍝# The starting state for a training step (t_rain<) from the weight
209⍝# arrays named on the right: the weights, zero averages, step 0.
210ᵐs̲tate< ← { @ weights →
211  ns ← ʰw̲ords weights
212  0 = t̲ally ns ? "bad-macro-argument right" ⎕R̲EJECT "name the starting weight arrays, one per dense layer"
213  ws ← ʰj̲oin '{ w → (d̲isclose w) c̲at ", " } m̲ap ns
214  zs ← ʰj̲oin '{ w → "0.0 * " c̲at (d̲isclose w) c̲at ", " } m̲ap ns
215  "(" c̲at ws c̲at zs c̲at zs c̲at "0.0)"
216}
217
218⍝## Counting and checking
219
220⍝# How many numbers the network of the spec on the left has to learn
221⍝# (weights and biases), worked out when the program is compiled: the
222⍝# call becomes that number. Nothing is written on the right: @.
223⍝# >> "net:" u_se< "Net"
224⍝# >> "2 16 relu 16 relu 3 softmax" net:p_arams< @
225⍝# 371
226ᵐp̲arams< ← { spec @ →
227  ws ← ʰw̲ords spec
228  0 < t̲ally ʰw̲rong ws ? "bad-macro-argument left" ⎕R̲EJECT ʰw̲rong ws
229  s ← ʰs̲izes ws
230  f̲ormat '+ r̲/ (1 + -1 d̲rop s) × 1 d̲rop s
231}
232
233⍝# Whether the weight arrays named on the right have the shapes the
234⍝# spec on the left says (a layer from m inputs to n outputs is
235⍝# m + 1 by n: the bias is the last row): 1 or 0.
236⍝# >> "net:" u_se< "Net"
237⍝# >> w1 := 3 4 r_eshape 0.0
238⍝# >> w2 := 5 3 r_eshape 0.0
239⍝# >> "2 4 tanh 3 softmax" net:s_hapes< "w1 w2"
240⍝# 1
241ᵐs̲hapes< ← { spec weights →
242  ws ← ʰw̲ords spec
243  ns ← ʰw̲ords weights
244  0 < t̲ally ʰw̲rong ws ? "bad-macro-argument left" ⎕R̲EJECT ʰw̲rong ws
245  s ← ʰs̲izes ws
246  0 < t̲ally ws ʰc̲ount ns ? "bad-macro-argument right" ⎕R̲EJECT ws ʰc̲ount ns
247  one ← '{ i → "((s_hape " c̲at (d̲isclose i s̲elect ns) c̲at ") m_atch " c̲at (f̲ormat 1 + i s̲elect s) c̲at " " c̲at (f̲ormat (i + 1) s̲elect s) c̲at ")" } m̲ap r̲ange t̲ally ns
248  ʰj̲oin '{ i → (i = 1) ? d̲isclose i s̲elect one◆ " & " c̲at d̲isclose i s̲elect one } m̲ap r̲ange t̲ally ns
249}