sourcedemos/pipelines/equations.xtl

1⍝!/usr/bin/env xetal 2⍝│ Pipelines of equations: stages that are Equations calls, each 3⍝│ reading the arrays it names (the previous value as it, earlier ones 4⍝│ by their labels), then the same layer by the libraries, a check that 5⍝│ they agree, and the pipeline drawn as math by flow:m_ath<: a row a 6⍝│ stage, an arrow carrying each value's name and shape, the equations 7⍝│ as Equations draws them. xetal expand equations.xtl shows what 8⍝│ each pipeline became. 9 10"flow:" u̲se< "Flow" 11"e:" u̲se< "Equations" 12"nn:" u̲se< "NN" 13"act:" u̲se< "Activations" 14"nm:" u̲se< "Norm" 15"at:" u̲se< "Attention" 16"k:" u̲se< "Check" 17 18⍝━━ Data 19 20⍝│ Four tokens of three features, and small fixed weights. 21n ← 4 22X ← 4 3 r̲eshape 0.5 × s̲in f̲loat r̲ange 12 23Wq ← 3 2 r̲eshape 0.4 × c̲os f̲loat r̲ange 6 24Wk ← 3 2 r̲eshape 0.4 × s̲in 2.0 × f̲loat r̲ange 6 25Wv ← 3 2 r̲eshape 0.6 × c̲os 3.0 × f̲loat r̲ange 6 26 27⍝━━ Attention as equations 28 29⍝│ One head of attention written as the paper writes it, a stage an 30⍝│ equation: the queries from the data (it), the keys and values from 31⍝│ it too, by its label x; the scores of each query against each key, 32⍝│ the softmax of each row written out, the values weighted. The 33⍝│ pipeline fans out from x and in at the scores and the last stage. 34head ← @ ᶠˡᵒʷp̲ipe< c#"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  x ← X
  q ← "Q[i,d] | n 2" ᵉe̲q< "sum c:=0..2 | x[i,c] * Wq[c,d]"
  k ← "K[j,d] | n 2" ᵉe̲q< "sum c:=0..2 | x[j,c] * Wk[c,d]"
  v ← "V[j,d] | n 2" ᵉe̲q< "sum c:=0..2 | x[j,c] * Wv[c,d]"
  s ← "S[i,j] | n n" ᵉe̲q< "(sum d:=0..1 | q[i,d] * k[j,d]) / sqrt(2.0)"
  _ ← { @ →
    ((s̲hape s) m̲atch r̲avel n c̲at n) ? @
    "shape" ⎕S̲IGNAL "stage 5 (s): shape " c̲at (f̲ormat s̲hape s) c̲at ", expected " c̲at f̲ormat r̲avel n c̲at n
  } @
  a ← "A[i,j] | n n" ᵉe̲q< "exp(s[i,j]) / (sum m:=0..n-1 | exp(s[i,m]))"
  g1:stage ← "Y[i,d] | n 2" ᵉe̲q< "sum j:=0..n-1 | a[i,j] * v[j,d]"
  g1:stage
} @)
ᵉe̲q< expands to
(({ @ →
  g4:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g3:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g2:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g5:grid ← n c̲at 2
  g6:g̲rid ← { k → g5:grid r̲eshape (o̲ffsets k s̲elect g5:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g5:grid) r̲eshape 0 }
  g5:grid r̲eshape (({ @ →
  g7:grid ← n c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g8:g̲rid ← { k → g7:grid r̲eshape (o̲ffsets k s̲elect g7:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g7:grid) r̲eshape 0 }
  '+ r̲/₃ g7:grid r̲eshape ((
    ((g2:r̲ank (x, "x", 2, "x[i,c]")) g4:a̲t ((
      ((x, "x", 2, "c") g3:c̲heck (0 + g8:g̲rid 3)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "i") g3:c̲heck (0 + g8:g̲rid 1))))) × 
    ((g2:r̲ank (Wq, "Wq", 2, "Wq[c,d]")) g4:a̲t ((
      ((Wq, "Wq", 2, "d") g3:c̲heck (0 + g8:g̲rid 2)) + (2 s̲elect s̲hape Wq) × 
      ((Wq, "Wq", 1, "c") g3:c̲heck (0 + g8:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g11:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g10:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g9:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g12:grid ← n c̲at 2
  g13:g̲rid ← { k → g12:grid r̲eshape (o̲ffsets k s̲elect g12:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g12:grid) r̲eshape 0 }
  g12:grid r̲eshape (({ @ →
  g14:grid ← n c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g15:g̲rid ← { k → g14:grid r̲eshape (o̲ffsets k s̲elect g14:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g14:grid) r̲eshape 0 }
  '+ r̲/₃ g14:grid r̲eshape ((
    ((g9:r̲ank (x, "x", 2, "x[j,c]")) g11:a̲t ((
      ((x, "x", 2, "c") g10:c̲heck (0 + g15:g̲rid 3)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "j") g10:c̲heck (0 + g15:g̲rid 1))))) × 
    ((g9:r̲ank (Wk, "Wk", 2, "Wk[c,d]")) g11:a̲t ((
      ((Wk, "Wk", 2, "d") g10:c̲heck (0 + g15:g̲rid 2)) + (2 s̲elect s̲hape Wk) × 
      ((Wk, "Wk", 1, "c") g10:c̲heck (0 + g15:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g18:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g17:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g16:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g19:grid ← n c̲at 2
  g20:g̲rid ← { k → g19:grid r̲eshape (o̲ffsets k s̲elect g19:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g19:grid) r̲eshape 0 }
  g19:grid r̲eshape (({ @ →
  g21:grid ← n c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g22:g̲rid ← { k → g21:grid r̲eshape (o̲ffsets k s̲elect g21:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g21:grid) r̲eshape 0 }
  '+ r̲/₃ g21:grid r̲eshape ((
    ((g16:r̲ank (x, "x", 2, "x[j,c]")) g18:a̲t ((
      ((x, "x", 2, "c") g17:c̲heck (0 + g22:g̲rid 3)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "j") g17:c̲heck (0 + g22:g̲rid 1))))) × 
    ((g16:r̲ank (Wv, "Wv", 2, "Wv[c,d]")) g18:a̲t ((
      ((Wv, "Wv", 2, "d") g17:c̲heck (0 + g22:g̲rid 2)) + (2 s̲elect s̲hape Wv) × 
      ((Wv, "Wv", 1, "c") g17:c̲heck (0 + g22:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g25:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g24:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g23:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g26:grid ← n c̲at n
  g27:g̲rid ← { k → g26:grid r̲eshape (o̲ffsets k s̲elect g26:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g26:grid) r̲eshape 0 }
  g26:grid r̲eshape ((({ @ →
  g28:grid ← n c̲at n c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..1" } (1 + 1 − 0))
  g29:g̲rid ← { k → g28:grid r̲eshape (o̲ffsets k s̲elect g28:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g28:grid) r̲eshape 0 }
  '+ r̲/₃ g28:grid r̲eshape ((
    ((g23:r̲ank (q, "q", 2, "q[i,d]")) g25:a̲t ((
      ((q, "q", 2, "d") g24:c̲heck (0 + g29:g̲rid 3)) + (2 s̲elect s̲hape q) × 
      ((q, "q", 1, "i") g24:c̲heck (0 + g29:g̲rid 1))))) × 
    ((g23:r̲ank (k, "k", 2, "k[j,d]")) g25:a̲t ((
      ((k, "k", 2, "d") g24:c̲heck (0 + g29:g̲rid 3)) + (2 s̲elect s̲hape k) × 
      ((k, "k", 1, "j") g24:c̲heck (0 + g29:g̲rid 2)))))))
} @) ÷ (2.0 ^ 0.5)))
} @))
ᵉe̲q< expands to
(({ @ →
  g74:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g73:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g72:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g75:grid ← n c̲at n
  g76:g̲rid ← { k → g75:grid r̲eshape (o̲ffsets k s̲elect g75:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g75:grid) r̲eshape 0 }
  g75:grid r̲eshape (((e̲xp 
    ((g72:r̲ank (s, "s", 2, "s[i,j]")) g74:a̲t ((
      ((s, "s", 2, "j") g73:c̲heck (0 + g76:g̲rid 2)) + (2 s̲elect s̲hape s) × 
      ((s, "s", 1, "i") g73:c̲heck (0 + g76:g̲rid 1)))))) ÷ ({ @ →
  g77:grid ← n c̲at n c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..n-1" } (1 + (n − 1) − 0))
  g78:g̲rid ← { k → g77:grid r̲eshape (o̲ffsets k s̲elect g77:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g77:grid) r̲eshape 0 }
  '+ r̲/₃ g77:grid r̲eshape ((e̲xp 
    ((g72:r̲ank (s, "s", 2, "s[i,m]")) g74:a̲t ((
      ((s, "s", 2, "m") g73:c̲heck (0 + g78:g̲rid 3)) + (2 s̲elect s̲hape s) × 
      ((s, "s", 1, "i") g73:c̲heck (0 + g78:g̲rid 1)))))))
} @)))
} @))
ᵉe̲q< expands to
(({ @ →
  g39:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g38:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g37:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g40:grid ← n c̲at 2
  g41:g̲rid ← { k → g40:grid r̲eshape (o̲ffsets k s̲elect g40:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g40:grid) r̲eshape 0 }
  g40:grid r̲eshape (({ @ →
  g42:grid ← n c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..n-1" } (1 + (n − 1) − 0))
  g43:g̲rid ← { k → g42:grid r̲eshape (o̲ffsets k s̲elect g42:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g42:grid) r̲eshape 0 }
  '+ r̲/₃ g42:grid r̲eshape ((
    ((g37:r̲ank (a, "a", 2, "a[i,j]")) g39:a̲t ((
      ((a, "a", 2, "j") g38:c̲heck (0 + g43:g̲rid 3)) + (2 s̲elect s̲hape a) × 
      ((a, "a", 1, "i") g38:c̲heck (0 + g43:g̲rid 1))))) × 
    ((g37:r̲ank (v, "v", 2, "v[j,d]")) g39:a̲t ((
      ((v, "v", 2, "d") g38:c̲heck (0 + g43:g̲rid 2)) + (2 s̲elect s̲hape v) × 
      ((v, "v", 1, "j") g38:c̲heck (0 + g43:g̲rid 3)))))))
} @))
} @))
35 X ▹x 36▷ "Q[i,d] | n 2" ᵉe̲q< "sum c:=0..2 | x[i,c] * Wq[c,d]" ▹q 37▷ "K[j,d] | n 2" ᵉe̲q< "sum c:=0..2 | x[j,c] * Wk[c,d]" ▹k 38▷ "V[j,d] | n 2" ᵉe̲q< "sum c:=0..2 | x[j,c] * Wv[c,d]" ▹v 39▷ "S[i,j] | n n" ᵉe̲q< "(sum d:=0..1 | q[i,d] * k[j,d]) / sqrt(2.0)" ▹s [n n] 40▷ "A[i,j] | n n" ᵉe̲q< "exp(s[i,j]) / (sum m:=0..n-1 | exp(s[i,m]))" ▹a 41▷ "Y[i,d] | n 2" ᵉe̲q< "sum j:=0..n-1 | a[i,j] * v[j,d]""# 42drawn ← @ ᶠˡᵒʷm̲ath< c#"
ᶠˡᵒʷm̲ath< expands to
((r#"\begin{aligned} "# c̲at r#"& "# c̲at r#"X"# c̲at r#" \qquad \triangleright \, x \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Q[i,d] | n 2" ᵉt̲ex< "sum c:=0..2 | x[i,c] * Wq[c,d]") c̲at r#" \qquad \triangleright \, q \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("K[j,d] | n 2" ᵉt̲ex< "sum c:=0..2 | x[j,c] * Wk[c,d]") c̲at r#" \qquad \triangleright \, k \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("V[j,d] | n 2" ᵉt̲ex< "sum c:=0..2 | x[j,c] * Wv[c,d]") c̲at r#" \qquad \triangleright \, v \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("S[i,j] | n n" ᵉt̲ex< "(sum d:=0..1 | q[i,d] * k[j,d]) / sqrt(2.0)") c̲at r#" \qquad \triangleright \, s: n \times n \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("A[i,j] | n n" ᵉt̲ex< "exp(s[i,j]) / (sum m:=0..n-1 | exp(s[i,m]))") c̲at r#" \qquad \triangleright \, a \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Y[i,d] | n 2" ᵉt̲ex< "sum j:=0..n-1 | a[i,j] * v[j,d]") c̲at r#""# c̲at r#" \end{aligned}"#))
ᵉt̲ex< expands to
("Q_{i,d} = \\sum_{c=0}^{2} x_{i,c} \\, Wq_{c,d}")
ᵉt̲ex< expands to
("K_{j,d} = \\sum_{c=0}^{2} x_{j,c} \\, Wk_{c,d}")
ᵉt̲ex< expands to
("V_{j,d} = \\sum_{c=0}^{2} x_{j,c} \\, Wv_{c,d}")
ᵉt̲ex< expands to
("S_{i,j} = \\frac{\\sum_{d=0}^{1} q_{i,d} \\, k_{j,d}}{\\sqrt{2.0}}")
ᵉt̲ex< expands to
("A_{i,j} = \\frac{\\exp(s_{i,j})}{\\sum_{m=0}^{n - 1} \\exp(s_{i,m})}")
ᵉt̲ex< expands to
("Y_{i,d} = \\sum_{j=0}^{n - 1} a_{i,j} \\, v_{j,d}")
43 X ▹x 44▷ "Q[i,d] | n 2" ᵉe̲q< "sum c:=0..2 | x[i,c] * Wq[c,d]" ▹q 45▷ "K[j,d] | n 2" ᵉe̲q< "sum c:=0..2 | x[j,c] * Wk[c,d]" ▹k 46▷ "V[j,d] | n 2" ᵉe̲q< "sum c:=0..2 | x[j,c] * Wv[c,d]" ▹v 47▷ "S[i,j] | n n" ᵉe̲q< "(sum d:=0..1 | q[i,d] * k[j,d]) / sqrt(2.0)" ▹s [n n] 48▷ "A[i,j] | n n" ᵉe̲q< "exp(s[i,j]) / (sum m:=0..n-1 | exp(s[i,m]))" ▹a 49▷ "Y[i,d] | n 2" ᵉe̲q< "sum j:=0..n-1 | a[i,j] * v[j,d]""# 50drawn 51head0 ← (Wq, Wk, Wv) ᵃᵗh̲ead (X, ᵃᵗa̲ll n) 52r1 ← "the equations are Attention's head, every token seeing every token" ᵏt̲est head ᵏn̲ear head0 53 54⍝━━ A BERT layer 55 56⍝│ One encoder layer of BERT (Tiny BERT, demos/micro-bert, applies two 57⍝│ of them to letters), every stage an equation: one head of attention 58⍝│ (queries, keys and values from the input h, the scores, the softmax, 59⍝│ the values weighted), out through Wo and added back to h; layer norm 60⍝│ (each row's mean and variance, then normalized, times the gain g, 61⍝│ plus beta); the feed-forward layer, a dense layer (its bias the last 62⍝│ row of W1), GELU, a dense layer added back to y, and layer norm 63⍝│ again. GELU is its tanh form written with exp, f / (1 + e^(-2u)) for 64⍝│ u = sqrt(2/pi) (f + 0.044715 f^3), which is Activations' g_elu (the 65⍝│ grammar has no tanh: ask E6). Checked against the layer written out 66⍝│ with the libraries; made-up weights, one head of width 3. 67Bq ← 3 3 r̲eshape 0.4 × s̲in 3.0 × f̲loat r̲ange 9 68Bk ← 3 3 r̲eshape 0.4 × c̲os 3.0 × f̲loat r̲ange 9 69Bv ← 3 3 r̲eshape 0.5 × s̲in 2.0 × f̲loat r̲ange 9 70Wo ← 3 3 r̲eshape 0.3 × c̲os 2.0 × f̲loat r̲ange 9 71W1 ← 4 6 r̲eshape 0.3 × s̲in f̲loat r̲ange 24 72W2 ← 7 3 r̲eshape 0.3 × c̲os f̲loat r̲ange 21 73g ← 1.0 0.9 1.1 74beta ← 0.1 0.0 -0.1 75layer ← @ ᶠˡᵒʷp̲ipe< c#"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  h ← X
  q ← "Q[i,d] | n 3" ᵉe̲q< "sum c:=0..2 | h[i,c] * Bq[c,d]"
  k ← "K[j,d] | n 3" ᵉe̲q< "sum c:=0..2 | h[j,c] * Bk[c,d]"
  v ← "V[j,d] | n 3" ᵉe̲q< "sum c:=0..2 | h[j,c] * Bv[c,d]"
  s ← "S[i,j] | n n" ᵉe̲q< "(sum d:=0..2 | q[i,d] * k[j,d]) / sqrt(3.0)"
  a ← "A[i,j] | n n" ᵉe̲q< "exp(s[i,j]) / (sum m:=0..n-1 | exp(s[i,m]))"
  z ← "Z[i,d] | n 3" ᵉe̲q< "sum j:=0..n-1 | a[i,j] * v[j,d]"
  r ← "R[i,c] | n 3" ᵉe̲q< "h[i,c] + (sum d:=0..2 | z[i,d] * Wo[d,c])"
  mu ← "Mean[i] | n" ᵉe̲q< "(sum c:=0..2 | r[i,c]) / 3.0"
  var ← "Var[i] | n" ᵉe̲q< "(sum c:=0..2 | (r[i,c] - mu[i])^2) / 3.0"
  y ← "N[i,c] | n 3" ᵉe̲q< "g[c] * (r[i,c] - mu[i]) / sqrt(var[i] + 0.00001) + beta[c]"
  _ ← { @ →
    ((s̲hape y) m̲atch r̲avel n c̲at 3) ? @
    "shape" ⎕S̲IGNAL "stage 11 (y): shape " c̲at (f̲ormat s̲hape y) c̲at ", expected " c̲at f̲ormat r̲avel n c̲at 3
  } @
  f ← "F[i,k] | n 6" ᵉe̲q< "(sum c:=0..2 | y[i,c] * W1[c,k]) + W1[3,k]"
  u ← "U[i,k] | n 6" ᵉe̲q< "0.7978845608028654 * (f[i,k] + 0.044715 * f[i,k]^3)"
  gl ← "G[i,k] | n 6" ᵉe̲q< "f[i,k] / (1 + exp(-2 * u[i,k]))"
  p ← "P[i,c] | n 3" ᵉe̲q< "y[i,c] + (sum k:=0..5 | gl[i,k] * W2[k,c]) + W2[6,c]"
  mu2 ← "Mean[i] | n" ᵉe̲q< "(sum c:=0..2 | p[i,c]) / 3.0"
  var2 ← "Var[i] | n" ᵉe̲q< "(sum c:=0..2 | (p[i,c] - mu2[i])^2) / 3.0"
  out ← "O[i,c] | n 3" ᵉe̲q< "g[c] * (p[i,c] - mu2[i]) / sqrt(var2[i] + 0.00001) + beta[c]"
  _ ← { @ →
    ((s̲hape out) m̲atch r̲avel n c̲at 3) ? @
    "shape" ⎕S̲IGNAL "stage 18 (out): shape " c̲at (f̲ormat s̲hape out) c̲at ", expected " c̲at f̲ormat r̲avel n c̲at 3
  } @
  out
} @)
ᵉe̲q< expands to
(({ @ →
  g46:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g45:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g44:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g47:grid ← n c̲at 3
  g48:g̲rid ← { k → g47:grid r̲eshape (o̲ffsets k s̲elect g47:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g47:grid) r̲eshape 0 }
  g47:grid r̲eshape (({ @ →
  g49:grid ← n c̲at 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g50:g̲rid ← { k → g49:grid r̲eshape (o̲ffsets k s̲elect g49:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g49:grid) r̲eshape 0 }
  '+ r̲/₃ g49:grid r̲eshape ((
    ((g44:r̲ank (h, "h", 2, "h[i,c]")) g46:a̲t ((
      ((h, "h", 2, "c") g45:c̲heck (0 + g50:g̲rid 3)) + (2 s̲elect s̲hape h) × 
      ((h, "h", 1, "i") g45:c̲heck (0 + g50:g̲rid 1))))) × 
    ((g44:r̲ank (Bq, "Bq", 2, "Bq[c,d]")) g46:a̲t ((
      ((Bq, "Bq", 2, "d") g45:c̲heck (0 + g50:g̲rid 2)) + (2 s̲elect s̲hape Bq) × 
      ((Bq, "Bq", 1, "c") g45:c̲heck (0 + g50:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g53:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g52:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g51:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g54:grid ← n c̲at 3
  g55:g̲rid ← { k → g54:grid r̲eshape (o̲ffsets k s̲elect g54:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g54:grid) r̲eshape 0 }
  g54:grid r̲eshape (({ @ →
  g56:grid ← n c̲at 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g57:g̲rid ← { k → g56:grid r̲eshape (o̲ffsets k s̲elect g56:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g56:grid) r̲eshape 0 }
  '+ r̲/₃ g56:grid r̲eshape ((
    ((g51:r̲ank (h, "h", 2, "h[j,c]")) g53:a̲t ((
      ((h, "h", 2, "c") g52:c̲heck (0 + g57:g̲rid 3)) + (2 s̲elect s̲hape h) × 
      ((h, "h", 1, "j") g52:c̲heck (0 + g57:g̲rid 1))))) × 
    ((g51:r̲ank (Bk, "Bk", 2, "Bk[c,d]")) g53:a̲t ((
      ((Bk, "Bk", 2, "d") g52:c̲heck (0 + g57:g̲rid 2)) + (2 s̲elect s̲hape Bk) × 
      ((Bk, "Bk", 1, "c") g52:c̲heck (0 + g57:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g60:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g59:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g58:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g61:grid ← n c̲at 3
  g62:g̲rid ← { k → g61:grid r̲eshape (o̲ffsets k s̲elect g61:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g61:grid) r̲eshape 0 }
  g61:grid r̲eshape (({ @ →
  g63:grid ← n c̲at 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g64:g̲rid ← { k → g63:grid r̲eshape (o̲ffsets k s̲elect g63:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g63:grid) r̲eshape 0 }
  '+ r̲/₃ g63:grid r̲eshape ((
    ((g58:r̲ank (h, "h", 2, "h[j,c]")) g60:a̲t ((
      ((h, "h", 2, "c") g59:c̲heck (0 + g64:g̲rid 3)) + (2 s̲elect s̲hape h) × 
      ((h, "h", 1, "j") g59:c̲heck (0 + g64:g̲rid 1))))) × 
    ((g58:r̲ank (Bv, "Bv", 2, "Bv[c,d]")) g60:a̲t ((
      ((Bv, "Bv", 2, "d") g59:c̲heck (0 + g64:g̲rid 2)) + (2 s̲elect s̲hape Bv) × 
      ((Bv, "Bv", 1, "c") g59:c̲heck (0 + g64:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g67:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g66:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g65:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g68:grid ← n c̲at n
  g69:g̲rid ← { k → g68:grid r̲eshape (o̲ffsets k s̲elect g68:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g68:grid) r̲eshape 0 }
  g68:grid r̲eshape ((({ @ →
  g70:grid ← n c̲at n c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g71:g̲rid ← { k → g70:grid r̲eshape (o̲ffsets k s̲elect g70:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g70:grid) r̲eshape 0 }
  '+ r̲/₃ g70:grid r̲eshape ((
    ((g65:r̲ank (q, "q", 2, "q[i,d]")) g67:a̲t ((
      ((q, "q", 2, "d") g66:c̲heck (0 + g71:g̲rid 3)) + (2 s̲elect s̲hape q) × 
      ((q, "q", 1, "i") g66:c̲heck (0 + g71:g̲rid 1))))) × 
    ((g65:r̲ank (k, "k", 2, "k[j,d]")) g67:a̲t ((
      ((k, "k", 2, "d") g66:c̲heck (0 + g71:g̲rid 3)) + (2 s̲elect s̲hape k) × 
      ((k, "k", 1, "j") g66:c̲heck (0 + g71:g̲rid 2)))))))
} @) ÷ (3.0 ^ 0.5)))
} @))
ᵉe̲q< expands to
(({ @ →
  g74:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g73:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g72:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g75:grid ← n c̲at n
  g76:g̲rid ← { k → g75:grid r̲eshape (o̲ffsets k s̲elect g75:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g75:grid) r̲eshape 0 }
  g75:grid r̲eshape (((e̲xp 
    ((g72:r̲ank (s, "s", 2, "s[i,j]")) g74:a̲t ((
      ((s, "s", 2, "j") g73:c̲heck (0 + g76:g̲rid 2)) + (2 s̲elect s̲hape s) × 
      ((s, "s", 1, "i") g73:c̲heck (0 + g76:g̲rid 1)))))) ÷ ({ @ →
  g77:grid ← n c̲at n c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..n-1" } (1 + (n − 1) − 0))
  g78:g̲rid ← { k → g77:grid r̲eshape (o̲ffsets k s̲elect g77:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g77:grid) r̲eshape 0 }
  '+ r̲/₃ g77:grid r̲eshape ((e̲xp 
    ((g72:r̲ank (s, "s", 2, "s[i,m]")) g74:a̲t ((
      ((s, "s", 2, "m") g73:c̲heck (0 + g78:g̲rid 3)) + (2 s̲elect s̲hape s) × 
      ((s, "s", 1, "i") g73:c̲heck (0 + g78:g̲rid 1)))))))
} @)))
} @))
ᵉe̲q< expands to
(({ @ →
  g81:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g80:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g79:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g82:grid ← n c̲at 3
  g83:g̲rid ← { k → g82:grid r̲eshape (o̲ffsets k s̲elect g82:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g82:grid) r̲eshape 0 }
  g82:grid r̲eshape (({ @ →
  g84:grid ← n c̲at 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..n-1" } (1 + (n − 1) − 0))
  g85:g̲rid ← { k → g84:grid r̲eshape (o̲ffsets k s̲elect g84:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g84:grid) r̲eshape 0 }
  '+ r̲/₃ g84:grid r̲eshape ((
    ((g79:r̲ank (a, "a", 2, "a[i,j]")) g81:a̲t ((
      ((a, "a", 2, "j") g80:c̲heck (0 + g85:g̲rid 3)) + (2 s̲elect s̲hape a) × 
      ((a, "a", 1, "i") g80:c̲heck (0 + g85:g̲rid 1))))) × 
    ((g79:r̲ank (v, "v", 2, "v[j,d]")) g81:a̲t ((
      ((v, "v", 2, "d") g80:c̲heck (0 + g85:g̲rid 2)) + (2 s̲elect s̲hape v) × 
      ((v, "v", 1, "j") g80:c̲heck (0 + g85:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g88:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g87:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g86:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g89:grid ← n c̲at 3
  g90:g̲rid ← { k → g89:grid r̲eshape (o̲ffsets k s̲elect g89:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g89:grid) r̲eshape 0 }
  g89:grid r̲eshape ((
    ((g86:r̲ank (h, "h", 2, "h[i,c]")) g88:a̲t ((
      ((h, "h", 2, "c") g87:c̲heck (0 + g90:g̲rid 2)) + (2 s̲elect s̲hape h) × 
      ((h, "h", 1, "i") g87:c̲heck (0 + g90:g̲rid 1))))) + ({ @ →
  g91:grid ← n c̲at 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g92:g̲rid ← { k → g91:grid r̲eshape (o̲ffsets k s̲elect g91:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g91:grid) r̲eshape 0 }
  '+ r̲/₃ g91:grid r̲eshape ((
    ((g86:r̲ank (z, "z", 2, "z[i,d]")) g88:a̲t ((
      ((z, "z", 2, "d") g87:c̲heck (0 + g92:g̲rid 3)) + (2 s̲elect s̲hape z) × 
      ((z, "z", 1, "i") g87:c̲heck (0 + g92:g̲rid 1))))) × 
    ((g86:r̲ank (Wo, "Wo", 2, "Wo[d,c]")) g88:a̲t ((
      ((Wo, "Wo", 2, "c") g87:c̲heck (0 + g92:g̲rid 2)) + (2 s̲elect s̲hape Wo) × 
      ((Wo, "Wo", 1, "d") g87:c̲heck (0 + g92:g̲rid 3)))))))
} @)))
} @))
ᵉe̲q< expands to
(({ @ →
  g95:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g94:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g93:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g96:grid ← n
  g97:g̲rid ← { k → g96:grid r̲eshape (o̲ffsets k s̲elect g96:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g96:grid) r̲eshape 0 }
  g96:grid r̲eshape ((({ @ →
  g98:grid ← n c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g99:g̲rid ← { k → g98:grid r̲eshape (o̲ffsets k s̲elect g98:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g98:grid) r̲eshape 0 }
  '+ r̲/₂ g98:grid r̲eshape (
    ((g93:r̲ank (r, "r", 2, "r[i,c]")) g95:a̲t ((
      ((r, "r", 2, "c") g94:c̲heck (0 + g99:g̲rid 2)) + (2 s̲elect s̲hape r) × 
      ((r, "r", 1, "i") g94:c̲heck (0 + g99:g̲rid 1))))))
} @) ÷ 3.0))
} @))
ᵉe̲q< expands to
(({ @ →
  g102:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g101:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g100:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g103:grid ← n
  g104:g̲rid ← { k → g103:grid r̲eshape (o̲ffsets k s̲elect g103:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g103:grid) r̲eshape 0 }
  g103:grid r̲eshape ((({ @ →
  g105:grid ← n c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g106:g̲rid ← { k → g105:grid r̲eshape (o̲ffsets k s̲elect g105:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g105:grid) r̲eshape 0 }
  '+ r̲/₂ g105:grid r̲eshape ((((
    ((g100:r̲ank (r, "r", 2, "r[i,c]")) g102:a̲t ((
      ((r, "r", 2, "c") g101:c̲heck (0 + g106:g̲rid 2)) + (2 s̲elect s̲hape r) × 
      ((r, "r", 1, "i") g101:c̲heck (0 + g106:g̲rid 1))))) − 
    ((g100:r̲ank (mu, "mu", 1, "mu[i]")) g102:a̲t (
      ((mu, "mu", 1, "i") g101:c̲heck (0 + g106:g̲rid 1)))))) ^ 2))
} @) ÷ 3.0))
} @))
ᵉe̲q< expands to
(({ @ →
  g109:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g108:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g107:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g110:grid ← n c̲at 3
  g111:g̲rid ← { k → g110:grid r̲eshape (o̲ffsets k s̲elect g110:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g110:grid) r̲eshape 0 }
  g110:grid r̲eshape ((((
    ((g107:r̲ank (g, "g", 1, "g[c]")) g109:a̲t (
      ((g, "g", 1, "c") g108:c̲heck (0 + g111:g̲rid 2)))) × ((
    ((g107:r̲ank (r, "r", 2, "r[i,c]")) g109:a̲t ((
      ((r, "r", 2, "c") g108:c̲heck (0 + g111:g̲rid 2)) + (2 s̲elect s̲hape r) × 
      ((r, "r", 1, "i") g108:c̲heck (0 + g111:g̲rid 1))))) − 
    ((g107:r̲ank (mu, "mu", 1, "mu[i]")) g109:a̲t (
      ((mu, "mu", 1, "i") g108:c̲heck (0 + g111:g̲rid 1))))))) ÷ ((
    ((g107:r̲ank (var, "var", 1, "var[i]")) g109:a̲t (
      ((var, "var", 1, "i") g108:c̲heck (0 + g111:g̲rid 1)))) + 0.00001) ^ 0.5)) + 
    ((g107:r̲ank (beta, "beta", 1, "beta[c]")) g109:a̲t (
      ((beta, "beta", 1, "c") g108:c̲heck (0 + g111:g̲rid 2))))))
} @))
ᵉe̲q< expands to
(({ @ →
  g114:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g113:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g112:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g115:grid ← n c̲at 6
  g116:g̲rid ← { k → g115:grid r̲eshape (o̲ffsets k s̲elect g115:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g115:grid) r̲eshape 0 }
  g115:grid r̲eshape ((({ @ →
  g117:grid ← n c̲at 6 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g118:g̲rid ← { k → g117:grid r̲eshape (o̲ffsets k s̲elect g117:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g117:grid) r̲eshape 0 }
  '+ r̲/₃ g117:grid r̲eshape ((
    ((g112:r̲ank (y, "y", 2, "y[i,c]")) g114:a̲t ((
      ((y, "y", 2, "c") g113:c̲heck (0 + g118:g̲rid 3)) + (2 s̲elect s̲hape y) × 
      ((y, "y", 1, "i") g113:c̲heck (0 + g118:g̲rid 1))))) × 
    ((g112:r̲ank (W1, "W1", 2, "W1[c,k]")) g114:a̲t ((
      ((W1, "W1", 2, "k") g113:c̲heck (0 + g118:g̲rid 2)) + (2 s̲elect s̲hape W1) × 
      ((W1, "W1", 1, "c") g113:c̲heck (0 + g118:g̲rid 3)))))))
} @) + 
    ((g112:r̲ank (W1, "W1", 2, "W1[3,k]")) g114:a̲t ((
      ((W1, "W1", 2, "k") g113:c̲heck (0 + g116:g̲rid 2)) + (2 s̲elect s̲hape W1) × 
      ((W1, "W1", 1, "3") g113:c̲heck 3))))))
} @))
ᵉe̲q< expands to
(({ @ →
  g121:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g120:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g119:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g122:grid ← n c̲at 6
  g123:g̲rid ← { k → g122:grid r̲eshape (o̲ffsets k s̲elect g122:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g122:grid) r̲eshape 0 }
  g122:grid r̲eshape ((0.7978845608028654 × ((
    ((g119:r̲ank (f, "f", 2, "f[i,k]")) g121:a̲t ((
      ((f, "f", 2, "k") g120:c̲heck (0 + g123:g̲rid 2)) + (2 s̲elect s̲hape f) × 
      ((f, "f", 1, "i") g120:c̲heck (0 + g123:g̲rid 1))))) + (0.044715 × (
    ((g119:r̲ank (f, "f", 2, "f[i,k]")) g121:a̲t ((
      ((f, "f", 2, "k") g120:c̲heck (0 + g123:g̲rid 2)) + (2 s̲elect s̲hape f) × 
      ((f, "f", 1, "i") g120:c̲heck (0 + g123:g̲rid 1))))) ^ 3))))))
} @))
ᵉe̲q< expands to
(({ @ →
  g126:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g125:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g124:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g127:grid ← n c̲at 6
  g128:g̲rid ← { k → g127:grid r̲eshape (o̲ffsets k s̲elect g127:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g127:grid) r̲eshape 0 }
  g127:grid r̲eshape ((
    ((g124:r̲ank (f, "f", 2, "f[i,k]")) g126:a̲t ((
      ((f, "f", 2, "k") g125:c̲heck (0 + g128:g̲rid 2)) + (2 s̲elect s̲hape f) × 
      ((f, "f", 1, "i") g125:c̲heck (0 + g128:g̲rid 1))))) ÷ ((1 + (e̲xp (0 − (2 × 
    ((g124:r̲ank (u, "u", 2, "u[i,k]")) g126:a̲t ((
      ((u, "u", 2, "k") g125:c̲heck (0 + g128:g̲rid 2)) + (2 s̲elect s̲hape u) × 
      ((u, "u", 1, "i") g125:c̲heck (0 + g128:g̲rid 1))))))))))))
} @))
ᵉe̲q< expands to
(({ @ →
  g131:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g130:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g129:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g132:grid ← n c̲at 3
  g133:g̲rid ← { k → g132:grid r̲eshape (o̲ffsets k s̲elect g132:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g132:grid) r̲eshape 0 }
  g132:grid r̲eshape (((
    ((g129:r̲ank (y, "y", 2, "y[i,c]")) g131:a̲t ((
      ((y, "y", 2, "c") g130:c̲heck (0 + g133:g̲rid 2)) + (2 s̲elect s̲hape y) × 
      ((y, "y", 1, "i") g130:c̲heck (0 + g133:g̲rid 1))))) + ({ @ →
  g134:grid ← n c̲at 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..5" } (1 + 5 − 0))
  g135:g̲rid ← { k → g134:grid r̲eshape (o̲ffsets k s̲elect g134:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g134:grid) r̲eshape 0 }
  '+ r̲/₃ g134:grid r̲eshape ((
    ((g129:r̲ank (gl, "gl", 2, "gl[i,k]")) g131:a̲t ((
      ((gl, "gl", 2, "k") g130:c̲heck (0 + g135:g̲rid 3)) + (2 s̲elect s̲hape gl) × 
      ((gl, "gl", 1, "i") g130:c̲heck (0 + g135:g̲rid 1))))) × 
    ((g129:r̲ank (W2, "W2", 2, "W2[k,c]")) g131:a̲t ((
      ((W2, "W2", 2, "c") g130:c̲heck (0 + g135:g̲rid 2)) + (2 s̲elect s̲hape W2) × 
      ((W2, "W2", 1, "k") g130:c̲heck (0 + g135:g̲rid 3)))))))
} @)) + 
    ((g129:r̲ank (W2, "W2", 2, "W2[6,c]")) g131:a̲t ((
      ((W2, "W2", 2, "c") g130:c̲heck (0 + g133:g̲rid 2)) + (2 s̲elect s̲hape W2) × 
      ((W2, "W2", 1, "6") g130:c̲heck 6))))))
} @))
ᵉe̲q< expands to
(({ @ →
  g138:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g137:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g136:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g139:grid ← n
  g140:g̲rid ← { k → g139:grid r̲eshape (o̲ffsets k s̲elect g139:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g139:grid) r̲eshape 0 }
  g139:grid r̲eshape ((({ @ →
  g141:grid ← n c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g142:g̲rid ← { k → g141:grid r̲eshape (o̲ffsets k s̲elect g141:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g141:grid) r̲eshape 0 }
  '+ r̲/₂ g141:grid r̲eshape (
    ((g136:r̲ank (p, "p", 2, "p[i,c]")) g138:a̲t ((
      ((p, "p", 2, "c") g137:c̲heck (0 + g142:g̲rid 2)) + (2 s̲elect s̲hape p) × 
      ((p, "p", 1, "i") g137:c̲heck (0 + g142:g̲rid 1))))))
} @) ÷ 3.0))
} @))
ᵉe̲q< expands to
(({ @ →
  g145:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g144:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g143:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g146:grid ← n
  g147:g̲rid ← { k → g146:grid r̲eshape (o̲ffsets k s̲elect g146:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g146:grid) r̲eshape 0 }
  g146:grid r̲eshape ((({ @ →
  g148:grid ← n c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g149:g̲rid ← { k → g148:grid r̲eshape (o̲ffsets k s̲elect g148:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g148:grid) r̲eshape 0 }
  '+ r̲/₂ g148:grid r̲eshape ((((
    ((g143:r̲ank (p, "p", 2, "p[i,c]")) g145:a̲t ((
      ((p, "p", 2, "c") g144:c̲heck (0 + g149:g̲rid 2)) + (2 s̲elect s̲hape p) × 
      ((p, "p", 1, "i") g144:c̲heck (0 + g149:g̲rid 1))))) − 
    ((g143:r̲ank (mu2, "mu2", 1, "mu2[i]")) g145:a̲t (
      ((mu2, "mu2", 1, "i") g144:c̲heck (0 + g149:g̲rid 1)))))) ^ 2))
} @) ÷ 3.0))
} @))
ᵉe̲q< expands to
(({ @ →
  g152:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g151:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g150:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g153:grid ← n c̲at 3
  g154:g̲rid ← { k → g153:grid r̲eshape (o̲ffsets k s̲elect g153:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g153:grid) r̲eshape 0 }
  g153:grid r̲eshape ((((
    ((g150:r̲ank (g, "g", 1, "g[c]")) g152:a̲t (
      ((g, "g", 1, "c") g151:c̲heck (0 + g154:g̲rid 2)))) × ((
    ((g150:r̲ank (p, "p", 2, "p[i,c]")) g152:a̲t ((
      ((p, "p", 2, "c") g151:c̲heck (0 + g154:g̲rid 2)) + (2 s̲elect s̲hape p) × 
      ((p, "p", 1, "i") g151:c̲heck (0 + g154:g̲rid 1))))) − 
    ((g150:r̲ank (mu2, "mu2", 1, "mu2[i]")) g152:a̲t (
      ((mu2, "mu2", 1, "i") g151:c̲heck (0 + g154:g̲rid 1))))))) ÷ ((
    ((g150:r̲ank (var2, "var2", 1, "var2[i]")) g152:a̲t (
      ((var2, "var2", 1, "i") g151:c̲heck (0 + g154:g̲rid 1)))) + 0.00001) ^ 0.5)) + 
    ((g150:r̲ank (beta, "beta", 1, "beta[c]")) g152:a̲t (
      ((beta, "beta", 1, "c") g151:c̲heck (0 + g154:g̲rid 2))))))
} @))
76 X ▹h 77▷ "Q[i,d] | n 3" ᵉe̲q< "sum c:=0..2 | h[i,c] * Bq[c,d]" ▹q 78▷ "K[j,d] | n 3" ᵉe̲q< "sum c:=0..2 | h[j,c] * Bk[c,d]" ▹k 79▷ "V[j,d] | n 3" ᵉe̲q< "sum c:=0..2 | h[j,c] * Bv[c,d]" ▹v 80▷ "S[i,j] | n n" ᵉe̲q< "(sum d:=0..2 | q[i,d] * k[j,d]) / sqrt(3.0)" ▹s 81▷ "A[i,j] | n n" ᵉe̲q< "exp(s[i,j]) / (sum m:=0..n-1 | exp(s[i,m]))" ▹a 82▷ "Z[i,d] | n 3" ᵉe̲q< "sum j:=0..n-1 | a[i,j] * v[j,d]" ▹z 83▷ "R[i,c] | n 3" ᵉe̲q< "h[i,c] + (sum d:=0..2 | z[i,d] * Wo[d,c])" ▹r 84▷ "Mean[i] | n" ᵉe̲q< "(sum c:=0..2 | r[i,c]) / 3.0" ▹mu 85▷ "Var[i] | n" ᵉe̲q< "(sum c:=0..2 | (r[i,c] - mu[i])^2) / 3.0" ▹var 86▷ "N[i,c] | n 3" ᵉe̲q< "g[c] * (r[i,c] - mu[i]) / sqrt(var[i] + 0.00001) + beta[c]" ▹y [n 3] 87▷ "F[i,k] | n 6" ᵉe̲q< "(sum c:=0..2 | y[i,c] * W1[c,k]) + W1[3,k]" ▹f 88▷ "U[i,k] | n 6" ᵉe̲q< "0.7978845608028654 * (f[i,k] + 0.044715 * f[i,k]^3)" ▹u 89▷ "G[i,k] | n 6" ᵉe̲q< "f[i,k] / (1 + exp(-2 * u[i,k]))" ▹gl 90▷ "P[i,c] | n 3" ᵉe̲q< "y[i,c] + (sum k:=0..5 | gl[i,k] * W2[k,c]) + W2[6,c]" ▹p 91▷ "Mean[i] | n" ᵉe̲q< "(sum c:=0..2 | p[i,c]) / 3.0" ▹mu2 92▷ "Var[i] | n" ᵉe̲q< "(sum c:=0..2 | (p[i,c] - mu2[i])^2) / 3.0" ▹var2 93▷ "O[i,c] | n 3" ᵉe̲q< "g[c] * (p[i,c] - mu2[i]) / sqrt(var2[i] + 0.00001) + beta[c]" ▹out [n 3]"# 94shown ← @ ᶠˡᵒʷm̲ath< c#"
ᶠˡᵒʷm̲ath< expands to
((r#"\begin{aligned} "# c̲at r#"& "# c̲at r#"X"# c̲at r#" \qquad \triangleright \, h \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Q[i,d] | n 3" ᵉt̲ex< "sum c:=0..2 | h[i,c] * Bq[c,d]") c̲at r#" \qquad \triangleright \, q \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("K[j,d] | n 3" ᵉt̲ex< "sum c:=0..2 | h[j,c] * Bk[c,d]") c̲at r#" \qquad \triangleright \, k \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("V[j,d] | n 3" ᵉt̲ex< "sum c:=0..2 | h[j,c] * Bv[c,d]") c̲at r#" \qquad \triangleright \, v \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("S[i,j] | n n" ᵉt̲ex< "(sum d:=0..2 | q[i,d] * k[j,d]) / sqrt(3.0)") c̲at r#" \qquad \triangleright \, s \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("A[i,j] | n n" ᵉt̲ex< "exp(s[i,j]) / (sum m:=0..n-1 | exp(s[i,m]))") c̲at r#" \qquad \triangleright \, a \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Z[i,d] | n 3" ᵉt̲ex< "sum j:=0..n-1 | a[i,j] * v[j,d]") c̲at r#" \qquad \triangleright \, z \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("R[i,c] | n 3" ᵉt̲ex< "h[i,c] + (sum d:=0..2 | z[i,d] * Wo[d,c])") c̲at r#" \qquad \triangleright \, r \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Mean[i] | n" ᵉt̲ex< "(sum c:=0..2 | r[i,c]) / 3.0") c̲at r#" \qquad \triangleright \, mu \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Var[i] | n" ᵉt̲ex< "(sum c:=0..2 | (r[i,c] - mu[i])^2) / 3.0") c̲at r#" \qquad \triangleright \, var \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("N[i,c] | n 3" ᵉt̲ex< "g[c] * (r[i,c] - mu[i]) / sqrt(var[i] + 0.00001) + beta[c]") c̲at r#" \qquad \triangleright \, y: n \times 3 \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("F[i,k] | n 6" ᵉt̲ex< "(sum c:=0..2 | y[i,c] * W1[c,k]) + W1[3,k]") c̲at r#" \qquad \triangleright \, f \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("U[i,k] | n 6" ᵉt̲ex< "0.7978845608028654 * (f[i,k] + 0.044715 * f[i,k]^3)") c̲at r#" \qquad \triangleright \, u \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("G[i,k] | n 6" ᵉt̲ex< "f[i,k] / (1 + exp(-2 * u[i,k]))") c̲at r#" \qquad \triangleright \, gl \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("P[i,c] | n 3" ᵉt̲ex< "y[i,c] + (sum k:=0..5 | gl[i,k] * W2[k,c]) + W2[6,c]") c̲at r#" \qquad \triangleright \, p \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Mean[i] | n" ᵉt̲ex< "(sum c:=0..2 | p[i,c]) / 3.0") c̲at r#" \qquad \triangleright \, mu2 \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Var[i] | n" ᵉt̲ex< "(sum c:=0..2 | (p[i,c] - mu2[i])^2) / 3.0") c̲at r#" \qquad \triangleright \, var2 \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("O[i,c] | n 3" ᵉt̲ex< "g[c] * (p[i,c] - mu2[i]) / sqrt(var2[i] + 0.00001) + beta[c]") c̲at r#" \qquad \triangleright \, out: n \times 3"# c̲at r#" \end{aligned}"#))
ᵉt̲ex< expands to
("Q_{i,d} = \\sum_{c=0}^{2} h_{i,c} \\, Bq_{c,d}")
ᵉt̲ex< expands to
("K_{j,d} = \\sum_{c=0}^{2} h_{j,c} \\, Bk_{c,d}")
ᵉt̲ex< expands to
("V_{j,d} = \\sum_{c=0}^{2} h_{j,c} \\, Bv_{c,d}")
ᵉt̲ex< expands to
("S_{i,j} = \\frac{\\sum_{d=0}^{2} q_{i,d} \\, k_{j,d}}{\\sqrt{3.0}}")
ᵉt̲ex< expands to
("A_{i,j} = \\frac{\\exp(s_{i,j})}{\\sum_{m=0}^{n - 1} \\exp(s_{i,m})}")
ᵉt̲ex< expands to
("Z_{i,d} = \\sum_{j=0}^{n - 1} a_{i,j} \\, v_{j,d}")
ᵉt̲ex< expands to
("R_{i,c} = h_{i,c} + \\left(\\sum_{d=0}^{2} z_{i,d} \\, Wo_{d,c}\\right)")
ᵉt̲ex< expands to
("Mean_{i} = \\frac{\\sum_{c=0}^{2} r_{i,c}}{3.0}")
ᵉt̲ex< expands to
("Var_{i} = \\frac{\\sum_{c=0}^{2} \\left(r_{i,c} - mu_{i}\\right)^{2}}{3.0}")
ᵉt̲ex< expands to
("N_{i,c} = \\frac{g_{c} \\, \\left(r_{i,c} - mu_{i}\\right)}{\\sqrt{var_{i} + 0.00001}} + beta_{c}")
ᵉt̲ex< expands to
("F_{i,k} = \\left(\\sum_{c=0}^{2} y_{i,c} \\, W1_{c,k}\\right) + W1_{3,k}")
ᵉt̲ex< expands to
("U_{i,k} = 0.7978845608028654 \\, \\left(f_{i,k} + 0.044715 \\, f_{i,k}^{3}\\right)")
ᵉt̲ex< expands to
("G_{i,k} = \\frac{f_{i,k}}{1 + \\exp(-2 \\, u_{i,k})}")
ᵉt̲ex< expands to
("P_{i,c} = y_{i,c} + \\left(\\sum_{k=0}^{5} gl_{i,k} \\, W2_{k,c}\\right) + W2_{6,c}")
ᵉt̲ex< expands to
("Mean_{i} = \\frac{\\sum_{c=0}^{2} p_{i,c}}{3.0}")
ᵉt̲ex< expands to
("Var_{i} = \\frac{\\sum_{c=0}^{2} \\left(p_{i,c} - mu2_{i}\\right)^{2}}{3.0}")
ᵉt̲ex< expands to
("O_{i,c} = \\frac{g_{c} \\, \\left(p_{i,c} - mu2_{i}\\right)}{\\sqrt{var2_{i} + 0.00001}} + beta_{c}")
95 X ▹h 96▷ "Q[i,d] | n 3" ᵉe̲q< "sum c:=0..2 | h[i,c] * Bq[c,d]" ▹q 97▷ "K[j,d] | n 3" ᵉe̲q< "sum c:=0..2 | h[j,c] * Bk[c,d]" ▹k 98▷ "V[j,d] | n 3" ᵉe̲q< "sum c:=0..2 | h[j,c] * Bv[c,d]" ▹v 99▷ "S[i,j] | n n" ᵉe̲q< "(sum d:=0..2 | q[i,d] * k[j,d]) / sqrt(3.0)" ▹s 100▷ "A[i,j] | n n" ᵉe̲q< "exp(s[i,j]) / (sum m:=0..n-1 | exp(s[i,m]))" ▹a 101▷ "Z[i,d] | n 3" ᵉe̲q< "sum j:=0..n-1 | a[i,j] * v[j,d]" ▹z 102▷ "R[i,c] | n 3" ᵉe̲q< "h[i,c] + (sum d:=0..2 | z[i,d] * Wo[d,c])" ▹r 103▷ "Mean[i] | n" ᵉe̲q< "(sum c:=0..2 | r[i,c]) / 3.0" ▹mu 104▷ "Var[i] | n" ᵉe̲q< "(sum c:=0..2 | (r[i,c] - mu[i])^2) / 3.0" ▹var 105▷ "N[i,c] | n 3" ᵉe̲q< "g[c] * (r[i,c] - mu[i]) / sqrt(var[i] + 0.00001) + beta[c]" ▹y [n 3] 106▷ "F[i,k] | n 6" ᵉe̲q< "(sum c:=0..2 | y[i,c] * W1[c,k]) + W1[3,k]" ▹f 107▷ "U[i,k] | n 6" ᵉe̲q< "0.7978845608028654 * (f[i,k] + 0.044715 * f[i,k]^3)" ▹u 108▷ "G[i,k] | n 6" ᵉe̲q< "f[i,k] / (1 + exp(-2 * u[i,k]))" ▹gl 109▷ "P[i,c] | n 3" ᵉe̲q< "y[i,c] + (sum k:=0..5 | gl[i,k] * W2[k,c]) + W2[6,c]" ▹p 110▷ "Mean[i] | n" ᵉe̲q< "(sum c:=0..2 | p[i,c]) / 3.0" ▹mu2 111▷ "Var[i] | n" ᵉe̲q< "(sum c:=0..2 | (p[i,c] - mu2[i])^2) / 3.0" ▹var2 112▷ "O[i,c] | n 3" ᵉe̲q< "g[c] * (p[i,c] - mu2[i]) / sqrt(var2[i] + 0.00001) + beta[c]" ▹out [n 3]"# 113shown 114a0 ← (g, beta) ⁿᵐl̲ayer X + ((Bq, Bk, Bv) ᵃᵗh̲ead (X, ᵃᵗa̲ll n)) '+ '× i̲nner Wo 115layer0 ← (g, beta) ⁿᵐl̲ayer a0 + (ᵃᶜᵗg̲elu a0 ⁿⁿd̲ense W1) ⁿⁿd̲ense W2 116r2 ← "the equations are the layer written out" ᵏt̲est layer ᵏn̲ear layer0 117 118⍝━━ The chain rule 119 120⍝│ A layer and its loss, then the loss's gradient back through it, in 121⍝│ one pipeline. Forward: z = x Wc, y = sigmoid(z) (written with exp: 122⍝│ Equations has no sigmoid yet, ask E6), the loss L half the squared 123⍝│ error against the targets T. Back: each stage the gradient before it 124⍝│ times its own stage's local derivative, the chain rule a row at a 125⍝│ time: dL/dy = y - T, dL/dz = dL/dy y (1 - y), dL/dWc the sum over the 126⍝│ batch of x times dL/dz. The local derivatives are written by hand 127⍝│ until Equations differentiates (ask E8). 128Wc ← 3 2 r̲eshape 0.5 × c̲os 2.0 × f̲loat r̲ange 6 129T ← 4 2 r̲eshape 0.0 1.0 1.0 0.0 0.0 1.0 1.0 0.0 130chain ← r#"\frac{\partial L}{\partial W} = \frac{\partial L}{\partial y} \cdot \frac{\partial y}{\partial z} \cdot \frac{\partial z}{\partial W}"# 131chain 132back ← @ ᶠˡᵒʷp̲ipe< c#"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  x ← X
  z ← "Z[i,k] | n 2" ᵉe̲q< "sum c:=0..2 | x[i,c] * Wc[c,k]"
  y ← "Y[i,k] | n 2" ᵉe̲q< "1 / (1 + exp(-z[i,k]))"
  l ← "L[] | " ᵉe̲q< "(sum i:=0..n-1, sum k:=0..1 | (y[i,k] - T[i,k])^2) / 2"
  gy ← "Gy[i,k] | n 2" ᵉe̲q< "y[i,k] - T[i,k]"
  gz ← "Gz[i,k] | n 2" ᵉe̲q< "gy[i,k] * y[i,k] * (1 - y[i,k])"
  gw ← "Gw[c,k] | 3 2" ᵉe̲q< "sum i:=0..n-1 | x[i,c] * gz[i,k]"
  _ ← { @ →
    ((s̲hape gw) m̲atch r̲avel 3 c̲at 2) ? @
    "shape" ⎕S̲IGNAL "stage 7 (gw): shape " c̲at (f̲ormat s̲hape gw) c̲at ", expected " c̲at f̲ormat r̲avel 3 c̲at 2
  } @
  gw
} @)
ᵉe̲q< expands to
(({ @ →
  g157:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g156:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g155:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g158:grid ← n c̲at 2
  g159:g̲rid ← { k → g158:grid r̲eshape (o̲ffsets k s̲elect g158:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g158:grid) r̲eshape 0 }
  g158:grid r̲eshape (({ @ →
  g160:grid ← n c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g161:g̲rid ← { k → g160:grid r̲eshape (o̲ffsets k s̲elect g160:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g160:grid) r̲eshape 0 }
  '+ r̲/₃ g160:grid r̲eshape ((
    ((g155:r̲ank (x, "x", 2, "x[i,c]")) g157:a̲t ((
      ((x, "x", 2, "c") g156:c̲heck (0 + g161:g̲rid 3)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "i") g156:c̲heck (0 + g161:g̲rid 1))))) × 
    ((g155:r̲ank (Wc, "Wc", 2, "Wc[c,k]")) g157:a̲t ((
      ((Wc, "Wc", 2, "k") g156:c̲heck (0 + g161:g̲rid 2)) + (2 s̲elect s̲hape Wc) × 
      ((Wc, "Wc", 1, "c") g156:c̲heck (0 + g161:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g164:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g163:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g162:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g165:grid ← n c̲at 2
  g166:g̲rid ← { k → g165:grid r̲eshape (o̲ffsets k s̲elect g165:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g165:grid) r̲eshape 0 }
  g165:grid r̲eshape ((1 ÷ ((1 + (e̲xp (0 − 
    ((g162:r̲ank (z, "z", 2, "z[i,k]")) g164:a̲t ((
      ((z, "z", 2, "k") g163:c̲heck (0 + g166:g̲rid 2)) + (2 s̲elect s̲hape z) × 
      ((z, "z", 1, "i") g163:c̲heck (0 + g166:g̲rid 1)))))))))))
} @))
ᵉe̲q< expands to
(({ @ →
  g169:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g168:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g167:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  ((({ @ →
  g172:grid ← ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..n-1" } (1 + (n − 1) − 0)) c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..1" } (1 + 1 − 0))
  g173:g̲rid ← { k → g172:grid r̲eshape (o̲ffsets k s̲elect g172:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g172:grid) r̲eshape 0 }
  '+ r̲/₁ '+ r̲/₂ g172:grid r̲eshape ((((
    ((g167:r̲ank (y, "y", 2, "y[i,k]")) g169:a̲t ((
      ((y, "y", 2, "k") g168:c̲heck (0 + g173:g̲rid 2)) + (2 s̲elect s̲hape y) × 
      ((y, "y", 1, "i") g168:c̲heck (0 + g173:g̲rid 1))))) − 
    ((g167:r̲ank (T, "T", 2, "T[i,k]")) g169:a̲t ((
      ((T, "T", 2, "k") g168:c̲heck (0 + g173:g̲rid 2)) + (2 s̲elect s̲hape T) × 
      ((T, "T", 1, "i") g168:c̲heck (0 + g173:g̲rid 1))))))) ^ 2))
} @) ÷ 2))
} @))
ᵉe̲q< expands to
(({ @ →
  g176:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g175:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g174:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g177:grid ← n c̲at 2
  g178:g̲rid ← { k → g177:grid r̲eshape (o̲ffsets k s̲elect g177:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g177:grid) r̲eshape 0 }
  g177:grid r̲eshape ((
    ((g174:r̲ank (y, "y", 2, "y[i,k]")) g176:a̲t ((
      ((y, "y", 2, "k") g175:c̲heck (0 + g178:g̲rid 2)) + (2 s̲elect s̲hape y) × 
      ((y, "y", 1, "i") g175:c̲heck (0 + g178:g̲rid 1))))) − 
    ((g174:r̲ank (T, "T", 2, "T[i,k]")) g176:a̲t ((
      ((T, "T", 2, "k") g175:c̲heck (0 + g178:g̲rid 2)) + (2 s̲elect s̲hape T) × 
      ((T, "T", 1, "i") g175:c̲heck (0 + g178:g̲rid 1)))))))
} @))
ᵉe̲q< expands to
(({ @ →
  g181:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g180:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g179:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g182:grid ← n c̲at 2
  g183:g̲rid ← { k → g182:grid r̲eshape (o̲ffsets k s̲elect g182:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g182:grid) r̲eshape 0 }
  g182:grid r̲eshape (((
    ((g179:r̲ank (gy, "gy", 2, "gy[i,k]")) g181:a̲t ((
      ((gy, "gy", 2, "k") g180:c̲heck (0 + g183:g̲rid 2)) + (2 s̲elect s̲hape gy) × 
      ((gy, "gy", 1, "i") g180:c̲heck (0 + g183:g̲rid 1))))) × 
    ((g179:r̲ank (y, "y", 2, "y[i,k]")) g181:a̲t ((
      ((y, "y", 2, "k") g180:c̲heck (0 + g183:g̲rid 2)) + (2 s̲elect s̲hape y) × 
      ((y, "y", 1, "i") g180:c̲heck (0 + g183:g̲rid 1)))))) × ((1 − 
    ((g179:r̲ank (y, "y", 2, "y[i,k]")) g181:a̲t ((
      ((y, "y", 2, "k") g180:c̲heck (0 + g183:g̲rid 2)) + (2 s̲elect s̲hape y) × 
      ((y, "y", 1, "i") g180:c̲heck (0 + g183:g̲rid 1)))))))))
} @))
ᵉe̲q< expands to
(({ @ →
  g186:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g185:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g184:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g187:grid ← 3 c̲at 2
  g188:g̲rid ← { k → g187:grid r̲eshape (o̲ffsets k s̲elect g187:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g187:grid) r̲eshape 0 }
  g187:grid r̲eshape (({ @ →
  g189:grid ← 3 c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..n-1" } (1 + (n − 1) − 0))
  g190:g̲rid ← { k → g189:grid r̲eshape (o̲ffsets k s̲elect g189:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g189:grid) r̲eshape 0 }
  '+ r̲/₃ g189:grid r̲eshape ((
    ((g184:r̲ank (x, "x", 2, "x[i,c]")) g186:a̲t ((
      ((x, "x", 2, "c") g185:c̲heck (0 + g190:g̲rid 1)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "i") g185:c̲heck (0 + g190:g̲rid 3))))) × 
    ((g184:r̲ank (gz, "gz", 2, "gz[i,k]")) g186:a̲t ((
      ((gz, "gz", 2, "k") g185:c̲heck (0 + g190:g̲rid 2)) + (2 s̲elect s̲hape gz) × 
      ((gz, "gz", 1, "i") g185:c̲heck (0 + g190:g̲rid 3)))))))
} @))
} @))
133 X ▹x 134▷ "Z[i,k] | n 2" ᵉe̲q< "sum c:=0..2 | x[i,c] * Wc[c,k]" ▹z 135▷ "Y[i,k] | n 2" ᵉe̲q< "1 / (1 + exp(-z[i,k]))" ▹y 136▷ "L[] | " ᵉe̲q< "(sum i:=0..n-1, sum k:=0..1 | (y[i,k] - T[i,k])^2) / 2" ▹l 137▷ "Gy[i,k] | n 2" ᵉe̲q< "y[i,k] - T[i,k]" ▹gy 138▷ "Gz[i,k] | n 2" ᵉe̲q< "gy[i,k] * y[i,k] * (1 - y[i,k])" ▹gz 139▷ "Gw[c,k] | 3 2" ᵉe̲q< "sum i:=0..n-1 | x[i,c] * gz[i,k]" ▹gw [3 2]"# 140backDrawn ← @ ᶠˡᵒʷm̲ath< c#"
ᶠˡᵒʷm̲ath< expands to
((r#"\begin{aligned} "# c̲at r#"& "# c̲at r#"X"# c̲at r#" \qquad \triangleright \, x \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Z[i,k] | n 2" ᵉt̲ex< "sum c:=0..2 | x[i,c] * Wc[c,k]") c̲at r#" \qquad \triangleright \, z \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Y[i,k] | n 2" ᵉt̲ex< "1 / (1 + exp(-z[i,k]))") c̲at r#" \qquad \triangleright \, y \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("L[] | " ᵉt̲ex< "(sum i:=0..n-1, sum k:=0..1 | (y[i,k] - T[i,k])^2) / 2") c̲at r#" \qquad \triangleright \, l \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Gy[i,k] | n 2" ᵉt̲ex< "y[i,k] - T[i,k]") c̲at r#" \qquad \triangleright \, gy \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Gz[i,k] | n 2" ᵉt̲ex< "gy[i,k] * y[i,k] * (1 - y[i,k])") c̲at r#" \qquad \triangleright \, gz \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Gw[c,k] | 3 2" ᵉt̲ex< "sum i:=0..n-1 | x[i,c] * gz[i,k]") c̲at r#" \qquad \triangleright \, gw: 3 \times 2"# c̲at r#" \end{aligned}"#))
ᵉt̲ex< expands to
("Z_{i,k} = \\sum_{c=0}^{2} x_{i,c} \\, Wc_{c,k}")
ᵉt̲ex< expands to
("Y_{i,k} = \\frac{1}{1 + \\exp(-z_{i,k})}")
ᵉt̲ex< expands to
("L = \\frac{\\sum_{i=0}^{n - 1} \\sum_{k=0}^{1} \\left(y_{i,k} - T_{i,k}\\right)^{2}}{2}")
ᵉt̲ex< expands to
("Gy_{i,k} = y_{i,k} - T_{i,k}")
ᵉt̲ex< expands to
("Gz_{i,k} = gy_{i,k} \\, y_{i,k} \\, \\left(1 - y_{i,k}\\right)")
ᵉt̲ex< expands to
("Gw_{c,k} = \\sum_{i=0}^{n - 1} x_{i,c} \\, gz_{i,k}")
141 X ▹x 142▷ "Z[i,k] | n 2" ᵉe̲q< "sum c:=0..2 | x[i,c] * Wc[c,k]" ▹z 143▷ "Y[i,k] | n 2" ᵉe̲q< "1 / (1 + exp(-z[i,k]))" ▹y 144▷ "L[] | " ᵉe̲q< "(sum i:=0..n-1, sum k:=0..1 | (y[i,k] - T[i,k])^2) / 2" ▹l 145▷ "Gy[i,k] | n 2" ᵉe̲q< "y[i,k] - T[i,k]" ▹gy 146▷ "Gz[i,k] | n 2" ᵉe̲q< "gy[i,k] * y[i,k] * (1 - y[i,k])" ▹gz 147▷ "Gw[c,k] | 3 2" ᵉe̲q< "sum i:=0..n-1 | x[i,c] * gz[i,k]" ▹gw [3 2]"# 148backDrawn 149⍝│ The same gradient written out (as the backprop demo writes it), and 150⍝│ by nudging each weight: the loss's change over twice the nudge. 151Y0 ← 1.0 ÷ 1.0 + e̲xp n̲eg X '+ '× i̲nner Wc 152back0 ← (o̲\ X) '+ '× i̲nner (Y0 − T) × Y0 × 1.0 − Y0 153ᵘl̲oss ← { w → 0.5 × '+ r̲/ r̲avel ((1.0 ÷ 1.0 + e̲xp n̲eg X '+ '× i̲nner w) − T) ^ 2 } 154ᵘn̲udge ← { j → 3 2 r̲eshape 0.000001 × f̲loat j = r̲ange 6 } 155nudged ← 3 2 r̲eshape '{ j → 500000.0 × (ᵘl̲oss Wc + ᵘn̲udge j) − ᵘl̲oss Wc − ᵘn̲udge j } e̲ach r̲ange 6 156r3 ← "the chain rule, stage by stage, is the gradient written out" ᵏt̲est back ᵏn̲ear back0 157r4 ← "and is what nudging each weight gives, within 1e-7" ᵏt̲est ᵏt̲rue 0.00000000000001 > (back − nudged) ^ 2 158 159ᵏr̲eport r1 ᵏa̲nd r2 ᵏa̲nd r3 ᵏa̲nd r4