programdemos/pipelines/pipelines.xtl

Pipelines on this repository's own libraries: each layer written as Flow writes data flow, left to right with |> between the stages, then the same layer in plain X_eTaL, then a check that they agree. xetal expand pipelines.xtl shows what each pipeline became.

source · imports flow: libs/Flow/src/Flow.xtlm; e: work/libs/Equations/src/Equations.xtl work/libs/Equations/src/Equations.xtlm; nn: libs/NN/src/NN.xtl; nm: libs/Norm/src/Norm.xtl; at: libs/Attention/src/Attention.xtl; k: work/libs/Check/src/Check.xtl work/libs/Check/src/Check.xtlm

Data

X : Float

value · line 18

A batch of 3 rows of 4 features; a dense layer's weights, 4 x 2, and its bias; small fixed values, so every result can be read.

X ← 3 4 r̲eshape 0.1 × f̲loat r̲ange 12

W : Float

value · line 19
W ← 4 2 r̲eshape 0.5 -0.25 0.75 0.1 -0.3 0.2 0.05 -0.6
Used in: dense, dense0, dense1

b : Float

value · line 20
b ← 0.1 -0.2
Used in: dense, dense0, dense1

Dense layer with its bias

dense : Float

value · line 26

The product, then the bias added along the last axis (NN's b_ias: X_eTaL has no implicit broadcasting, so the axis is the library's).

dense ← @ ᶠˡᵒʷp̲ipe< c"X ▷ it '+ '× i̲nner W ▷ b ⁿⁿb̲ias it"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  g1:stage ← X
  g2:stage ← { @ →
    it ← g1:stage
    it '+ '× i̲nner W
  } @
  g3:stage ← { @ →
    it ← g2:stage
    b ⁿⁿb̲ias it
  } @
  g3:stage
} @)
Used in: c1

dense0 : Float

value · line 27
dense0 ← (X '+ '× i̲nner W) + 3 2 r̲eshape b
Used in: c1, c1b

c1 : Char

value · line 28
c1 ← "dense, its bias" ᵏt̲est dense ᵏn̲ear dense0

dense1 : Float

value · line 30

As math: the product, then the bias, a stage each.

dense1 ← @ ᶠˡᵒʷp̲ipe< c#"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  g4:stage ← X
  p ← { @ →
    it ← g4:stage
    "P[i,o] | 3 2" ᵉe̲q< "sum j:=0..3 | it[i,j] * W[j,o]"
  } @
  g5:stage ← "Y[i,o] | 3 2" ᵉe̲q< "p[i,o] + b[o]"
  g5:stage
} @)
ᵉe̲q< expands to
({ @ →
  g8:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g7:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g6:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g9:grid ← 3 c̲at 2
  g10:g̲rid ← { k → g9:grid r̲eshape (o̲ffsets k s̲elect g9:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g9:grid) r̲eshape 0 }
  g9:grid r̲eshape (({ @ →
  g11:grid ← 3 c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..3" } (1 + 3 − 0))
  g12:g̲rid ← { k → g11:grid r̲eshape (o̲ffsets k s̲elect g11:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g11:grid) r̲eshape 0 }
  '+ r̲/₃ g11:grid r̲eshape ((
    ((g6:r̲ank (it, "it", 2, "it[i,j]")) g8:a̲t ((
      ((it, "it", 2, "j") g7:c̲heck (0 + g12:g̲rid 3)) + (2 s̲elect s̲hape it) × 
      ((it, "it", 1, "i") g7:c̲heck (0 + g12:g̲rid 1))))) × 
    ((g6:r̲ank (W, "W", 2, "W[j,o]")) g8:a̲t ((
      ((W, "W", 2, "o") g7:c̲heck (0 + g12:g̲rid 2)) + (2 s̲elect s̲hape W) × 
      ((W, "W", 1, "j") g7:c̲heck (0 + g12:g̲rid 3)))))))
} @))
} @)
ᵉe̲q< expands to
(({ @ →
  g15:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g14:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g13:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g16:grid ← 3 c̲at 2
  g17:g̲rid ← { k → g16:grid r̲eshape (o̲ffsets k s̲elect g16:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g16:grid) r̲eshape 0 }
  g16:grid r̲eshape ((
    ((g13:r̲ank (p, "p", 2, "p[i,o]")) g15:a̲t ((
      ((p, "p", 2, "o") g14:c̲heck (0 + g17:g̲rid 2)) + (2 s̲elect s̲hape p) × 
      ((p, "p", 1, "i") g14:c̲heck (0 + g17:g̲rid 1))))) + 
    ((g13:r̲ank (b, "b", 1, "b[o]")) g15:a̲t (
      ((b, "b", 1, "o") g14:c̲heck (0 + g17:g̲rid 2))))))
} @))
Used in: c1b

denseM : Char

value · line 34
denseM ← @ ᶠˡᵒʷm̲ath< c#"
ᶠˡᵒʷm̲ath< expands to
((r#"\begin{aligned} "# c̲at r#"& "# c̲at r#"X"# c̲at r#" \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("P[i,o] | 3 2" ᵉt̲ex< "sum j:=0..3 | X[i,j] * W[j,o]") c̲at r#" \qquad \triangleright \, p \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Y[i,o] | 3 2" ᵉt̲ex< "p[i,o] + b[o]") c̲at r#""# c̲at r#" \end{aligned}"#))
ᵉt̲ex< expands to
("P_{i,o} = \\sum_{j=0}^{3} X_{i,j} \\, W_{j,o}")
ᵉt̲ex< expands to
("Y_{i,o} = p_{i,o} + b_{o}")

c1b : Char

value · line 39
c1b ← "the same as equations" ᵏt̲est dense1 ᵏn̲ear dense0

Stable softmax

z : Float

value · line 45

The largest subtracted before e_xp, so e_xp never overflows: the labels $m and $e name stages that later stages use.

z ← 1000.0 1001.0 1002.0
Used in: soft, soft0, soft1

soft : Float

value · line 46
soft ← @ ᶠˡᵒʷp̲ipe< c"z ▷ 'm̲ax r̲/ it ▹m ▷ e̲xp z − m ▹e ▷ e ÷ '+ r̲/ e"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  g18:stage ← z
  m ← { @ →
    it ← g18:stage
    'm̲ax r̲/ it
  } @
  e ← e̲xp z − m
  g19:stage ← e ÷ '+ r̲/ e
  g19:stage
} @)
Used in: c2

soft0 : Float

value · line 47
soft0 ← ⁿⁿs̲oftmax z
Used in: c2, c2b

c2 : Char

value · line 48
c2 ← "stable softmax" ᵏt̲est soft ᵏn̲ear soft0

soft1 : Float

value · line 51

As math: the exponential of each less the largest (max as a reduction), each over their sum.

soft1 ← @ ᶠˡᵒʷp̲ipe< c#"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  g20:stage ← z
  e ← { @ →
    it ← g20:stage
    "E[j] | 3" ᵉe̲q< "exp(it[j] - (max i:=0..2 | it[i]))"
  } @
  g21:stage ← "P[j] | 3" ᵉe̲q< "e[j] / (sum i:=0..2 | e[i])"
  g21:stage
} @)
ᵉe̲q< expands to
({ @ →
  g24:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g23:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g22:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g25:grid ← 3
  g26:g̲rid ← { k → g25:grid r̲eshape (o̲ffsets k s̲elect g25:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g25:grid) r̲eshape 0 }
  g25:grid r̲eshape ((e̲xp (
    ((g22:r̲ank (it, "it", 1, "it[j]")) g24:a̲t (
      ((it, "it", 1, "j") g23:c̲heck (0 + g26:g̲rid 1)))) − ({ @ →
  g27:grid ← 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "max's range runs low..high: 0..2" } (1 + 2 − 0))
  g28:g̲rid ← { k → g27:grid r̲eshape (o̲ffsets k s̲elect g27:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g27:grid) r̲eshape 0 }
  'm̲ax r̲/₂ g27:grid r̲eshape (
    ((g22:r̲ank (it, "it", 1, "it[i]")) g24:a̲t (
      ((it, "it", 1, "i") g23:c̲heck (0 + g28:g̲rid 2)))))
} @))))
} @)
ᵉe̲q< expands to
(({ @ →
  g31:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g30:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g29:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g32:grid ← 3
  g33:g̲rid ← { k → g32:grid r̲eshape (o̲ffsets k s̲elect g32:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g32:grid) r̲eshape 0 }
  g32:grid r̲eshape ((
    ((g29:r̲ank (e, "e", 1, "e[j]")) g31:a̲t (
      ((e, "e", 1, "j") g30:c̲heck (0 + g33:g̲rid 1)))) ÷ ({ @ →
  g34:grid ← 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g35:g̲rid ← { k → g34:grid r̲eshape (o̲ffsets k s̲elect g34:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g34:grid) r̲eshape 0 }
  '+ r̲/₂ g34:grid r̲eshape (
    ((g29:r̲ank (e, "e", 1, "e[i]")) g31:a̲t (
      ((e, "e", 1, "i") g30:c̲heck (0 + g35:g̲rid 2)))))
} @)))
} @))
Used in: c2b

softM : Char

value · line 55
softM ← @ ᶠˡᵒʷm̲ath< c#"
ᶠˡᵒʷm̲ath< expands to
((r#"\begin{aligned} "# c̲at r#"& "# c̲at r#"z"# c̲at r#" \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("E[j] | 3" ᵉt̲ex< "exp(z[j] - (max i:=0..2 | z[i]))") c̲at r#" \qquad \triangleright \, e \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("P[j] | 3" ᵉt̲ex< "e[j] / (sum i:=0..2 | e[i])") c̲at r#""# c̲at r#" \end{aligned}"#))
ᵉt̲ex< expands to
("E_{j} = \\exp(z_{j} - \\left(\\max_{i=0}^{2} z_{i}\\right))")
ᵉt̲ex< expands to
("P_{j} = \\frac{e_{j}}{\\sum_{i=0}^{2} e_{i}}")

c2b : Char

value · line 60
c2b ← "the same as equations" ᵏt̲est soft1 ᵏn̲ear soft0

Layer norm

g : Float

value · line 66

Each row standardized (less its mean, over its standard deviation: Norm's s_tandard), then times the gain, plus the bias.

g ← 1.0 2.0 1.0 0.5
Used in: norm, norm0, norm1

c : Float

value · line 67
c ← 0.0 0.1 0.0 -0.1
Used in: norm, norm0, norm1

norm : Float

value · line 68
norm ← @ ᶠˡᵒʷp̲ipe< c"X ▷ ⁿᵐs̲tandard ▷ it × (s̲hape it) r̲eshape g ▷ it + (s̲hape it) r̲eshape c"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  g36:stage ← X
  g37:stage ← ⁿᵐs̲tandard g36:stage
  g38:stage ← { @ →
    it ← g37:stage
    it × (s̲hape it) r̲eshape g
  } @
  g39:stage ← { @ →
    it ← g38:stage
    it + (s̲hape it) r̲eshape c
  } @
  g39:stage
} @)
Used in: c3

norm0 : Float

value · line 69
norm0 ← (g, c) ⁿᵐl̲ayer X
Used in: c3, c3b

c3 : Char

value · line 70
c3 ← "layer norm" ᵏt̲est norm ᵏn̲ear norm0

norm1 : Float

value · line 73

As math: each row's mean and variance, then each item normalized, times the gain, plus the bias.

norm1 ← @ ᶠˡᵒʷp̲ipe< c#"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  x ← X
  mu ← { @ →
    it ← x
    "Mean[i] | 3" ᵉe̲q< "(sum j:=0..3 | it[i,j]) / 4.0"
  } @
  var ← "Var[i] | 3" ᵉe̲q< "(sum j:=0..3 | (x[i,j] - mu[i])^2) / 4.0"
  g40:stage ← "Y[i,j] | 3 4" ᵉe̲q< "g[j] * (x[i,j] - mu[i]) / sqrt(var[i] + 0.00001) + c[j]"
  g40:stage
} @)
ᵉe̲q< expands to
({ @ →
  g43:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g42:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g41:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g44:grid ← 3
  g45:g̲rid ← { k → g44:grid r̲eshape (o̲ffsets k s̲elect g44:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g44:grid) r̲eshape 0 }
  g44:grid r̲eshape ((({ @ →
  g46:grid ← 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..3" } (1 + 3 − 0))
  g47:g̲rid ← { k → g46:grid r̲eshape (o̲ffsets k s̲elect g46:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g46:grid) r̲eshape 0 }
  '+ r̲/₂ g46:grid r̲eshape (
    ((g41:r̲ank (it, "it", 2, "it[i,j]")) g43:a̲t ((
      ((it, "it", 2, "j") g42:c̲heck (0 + g47:g̲rid 2)) + (2 s̲elect s̲hape it) × 
      ((it, "it", 1, "i") g42:c̲heck (0 + g47:g̲rid 1))))))
} @) ÷ 4.0))
} @)
ᵉe̲q< expands to
(({ @ →
  g50:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g49:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g48:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g51:grid ← 3
  g52:g̲rid ← { k → g51:grid r̲eshape (o̲ffsets k s̲elect g51:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g51:grid) r̲eshape 0 }
  g51:grid r̲eshape ((({ @ →
  g53:grid ← 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..3" } (1 + 3 − 0))
  g54:g̲rid ← { k → g53:grid r̲eshape (o̲ffsets k s̲elect g53:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g53:grid) r̲eshape 0 }
  '+ r̲/₂ g53:grid r̲eshape ((((
    ((g48:r̲ank (x, "x", 2, "x[i,j]")) g50:a̲t ((
      ((x, "x", 2, "j") g49:c̲heck (0 + g54:g̲rid 2)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "i") g49:c̲heck (0 + g54:g̲rid 1))))) − 
    ((g48:r̲ank (mu, "mu", 1, "mu[i]")) g50:a̲t (
      ((mu, "mu", 1, "i") g49:c̲heck (0 + g54:g̲rid 1)))))) ^ 2))
} @) ÷ 4.0))
} @))
ᵉe̲q< expands to
(({ @ →
  g57:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g56:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g55:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g58:grid ← 3 c̲at 4
  g59:g̲rid ← { k → g58:grid r̲eshape (o̲ffsets k s̲elect g58:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g58:grid) r̲eshape 0 }
  g58:grid r̲eshape ((((
    ((g55:r̲ank (g, "g", 1, "g[j]")) g57:a̲t (
      ((g, "g", 1, "j") g56:c̲heck (0 + g59:g̲rid 2)))) × ((
    ((g55:r̲ank (x, "x", 2, "x[i,j]")) g57:a̲t ((
      ((x, "x", 2, "j") g56:c̲heck (0 + g59:g̲rid 2)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "i") g56:c̲heck (0 + g59:g̲rid 1))))) − 
    ((g55:r̲ank (mu, "mu", 1, "mu[i]")) g57:a̲t (
      ((mu, "mu", 1, "i") g56:c̲heck (0 + g59:g̲rid 1))))))) ÷ ((
    ((g55:r̲ank (var, "var", 1, "var[i]")) g57:a̲t (
      ((var, "var", 1, "i") g56:c̲heck (0 + g59:g̲rid 1)))) + 0.00001) ^ 0.5)) + 
    ((g55:r̲ank (c, "c", 1, "c[j]")) g57:a̲t (
      ((c, "c", 1, "j") g56:c̲heck (0 + g59:g̲rid 2))))))
} @))
Used in: c3b

normM : Char

value · line 78
normM ← @ ᶠˡᵒʷm̲ath< c#"
ᶠˡᵒʷm̲ath< expands to
((r#"\begin{aligned} "# c̲at r#"& "# c̲at r#"X"# c̲at r#" \qquad \triangleright \, x \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Mean[i] | 3" ᵉt̲ex< "(sum j:=0..3 | x[i,j]) / 4.0") c̲at r#" \qquad \triangleright \, mu \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Var[i] | 3" ᵉt̲ex< "(sum j:=0..3 | (x[i,j] - mu[i])^2) / 4.0") c̲at r#" \qquad \triangleright \, var \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Y[i,j] | 3 4" ᵉt̲ex< "g[j] * (x[i,j] - mu[i]) / sqrt(var[i] + 0.00001) + c[j]") c̲at r#""# c̲at r#" \end{aligned}"#))
ᵉt̲ex< expands to
("Mean_{i} = \\frac{\\sum_{j=0}^{3} x_{i,j}}{4.0}")
ᵉt̲ex< expands to
("Var_{i} = \\frac{\\sum_{j=0}^{3} \\left(x_{i,j} - mu_{i}\\right)^{2}}{4.0}")
ᵉt̲ex< expands to
("Y_{i,j} = \\frac{g_{j} \\, \\left(x_{i,j} - mu_{i}\\right)}{\\sqrt{var_{i} + 0.00001}} + c_{j}")

c3b : Char

value · line 84
c3b ← "the same as equations" ᵏt̲est norm1 ᵏn̲ear norm0

Attention

Wq : Float

value · line 92

One head: queries, keys and values from the same rows, taken apart by a tuple label, then the scores, their weights under a causal mask and the mix of the values. The verbatim form c#"..."# keeps the transpose o_\ as written.

Wq ← 4 2 r̲eshape 0.2 0.1 -0.1 0.3 0.05 0.2 0.1 -0.2
Used in: head, head0, head1

Wk : Float

value · line 93
Wk ← 4 2 r̲eshape 0.1 0.2 0.3 -0.1 -0.2 0.1 0.2 0.05
Used in: head, head0, head1

Wv : Float

value · line 94
Wv ← 4 2 r̲eshape 1.0 0.0 0.0 1.0 0.5 0.5 -0.5 0.5
Used in: head, head0, head1

M : Float

value · line 95
M ← ᵃᵗc̲ausal 3
Used in: head, head0, head1

head0 : Float

value · line 102
head0 ← (Wq, Wk, Wv) ᵃᵗh̲ead (X, M)
Used in: c4, c4b

c4 : Char

value · line 103
c4 ← "attention, one head" ᵏt̲est head ᵏn̲ear head0

head1 : Float

value · line 107

As math: the queries, keys and values, the scores, the softmax under the causal mask (M multiplies each exponential: a masked place gets 0), the values weighted.

head1 ← @ ᶠˡᵒʷp̲ipe< c#"
ᶠˡᵒʷp̲ipe< expands to
({ @ →
  x ← X
  q ← "Q[i,d] | 3 2" ᵉe̲q< "sum f:=0..3 | x[i,f] * Wq[f,d]"
  k ← "K[j,d] | 3 2" ᵉe̲q< "sum f:=0..3 | x[j,f] * Wk[f,d]"
  v ← "V[j,d] | 3 2" ᵉe̲q< "sum f:=0..3 | x[j,f] * Wv[f,d]"
  s ← "S[i,j] | 3 3" ᵉe̲q< "(sum d:=0..1 | q[i,d] * k[j,d]) / sqrt(2.0)"
  a ← "A[i,j] | 3 3" ᵉe̲q< "M[i,j] * exp(s[i,j]) / (sum m:=0..2 | M[i,m] * exp(s[i,m]))"
  g63:stage ← "Y[i,d] | 3 2" ᵉe̲q< "sum j:=0..2 | a[i,j] * v[j,d]"
  g63:stage
} @)
ᵉe̲q< expands to
(({ @ →
  g66:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g65:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g64:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g67:grid ← 3 c̲at 2
  g68:g̲rid ← { k → g67:grid r̲eshape (o̲ffsets k s̲elect g67:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g67:grid) r̲eshape 0 }
  g67:grid r̲eshape (({ @ →
  g69:grid ← 3 c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..3" } (1 + 3 − 0))
  g70:g̲rid ← { k → g69:grid r̲eshape (o̲ffsets k s̲elect g69:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g69:grid) r̲eshape 0 }
  '+ r̲/₃ g69:grid r̲eshape ((
    ((g64:r̲ank (x, "x", 2, "x[i,f]")) g66:a̲t ((
      ((x, "x", 2, "f") g65:c̲heck (0 + g70:g̲rid 3)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "i") g65:c̲heck (0 + g70:g̲rid 1))))) × 
    ((g64:r̲ank (Wq, "Wq", 2, "Wq[f,d]")) g66:a̲t ((
      ((Wq, "Wq", 2, "d") g65:c̲heck (0 + g70:g̲rid 2)) + (2 s̲elect s̲hape Wq) × 
      ((Wq, "Wq", 1, "f") g65:c̲heck (0 + g70:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g73:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g72:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g71:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g74:grid ← 3 c̲at 2
  g75:g̲rid ← { k → g74:grid r̲eshape (o̲ffsets k s̲elect g74:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g74:grid) r̲eshape 0 }
  g74:grid r̲eshape (({ @ →
  g76:grid ← 3 c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..3" } (1 + 3 − 0))
  g77:g̲rid ← { k → g76:grid r̲eshape (o̲ffsets k s̲elect g76:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g76:grid) r̲eshape 0 }
  '+ r̲/₃ g76:grid r̲eshape ((
    ((g71:r̲ank (x, "x", 2, "x[j,f]")) g73:a̲t ((
      ((x, "x", 2, "f") g72:c̲heck (0 + g77:g̲rid 3)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "j") g72:c̲heck (0 + g77:g̲rid 1))))) × 
    ((g71:r̲ank (Wk, "Wk", 2, "Wk[f,d]")) g73:a̲t ((
      ((Wk, "Wk", 2, "d") g72:c̲heck (0 + g77:g̲rid 2)) + (2 s̲elect s̲hape Wk) × 
      ((Wk, "Wk", 1, "f") g72:c̲heck (0 + g77:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g80:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g79:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g78:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g81:grid ← 3 c̲at 2
  g82:g̲rid ← { k → g81:grid r̲eshape (o̲ffsets k s̲elect g81:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g81:grid) r̲eshape 0 }
  g81:grid r̲eshape (({ @ →
  g83:grid ← 3 c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..3" } (1 + 3 − 0))
  g84:g̲rid ← { k → g83:grid r̲eshape (o̲ffsets k s̲elect g83:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g83:grid) r̲eshape 0 }
  '+ r̲/₃ g83:grid r̲eshape ((
    ((g78:r̲ank (x, "x", 2, "x[j,f]")) g80:a̲t ((
      ((x, "x", 2, "f") g79:c̲heck (0 + g84:g̲rid 3)) + (2 s̲elect s̲hape x) × 
      ((x, "x", 1, "j") g79:c̲heck (0 + g84:g̲rid 1))))) × 
    ((g78:r̲ank (Wv, "Wv", 2, "Wv[f,d]")) g80:a̲t ((
      ((Wv, "Wv", 2, "d") g79:c̲heck (0 + g84:g̲rid 2)) + (2 s̲elect s̲hape Wv) × 
      ((Wv, "Wv", 1, "f") g79:c̲heck (0 + g84:g̲rid 3)))))))
} @))
} @))
ᵉe̲q< expands to
(({ @ →
  g87:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g86:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g85:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g88:grid ← 3 c̲at 3
  g89:g̲rid ← { k → g88:grid r̲eshape (o̲ffsets k s̲elect g88:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g88:grid) r̲eshape 0 }
  g88:grid r̲eshape ((({ @ →
  g90:grid ← 3 c̲at 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..1" } (1 + 1 − 0))
  g91:g̲rid ← { k → g90:grid r̲eshape (o̲ffsets k s̲elect g90:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g90:grid) r̲eshape 0 }
  '+ r̲/₃ g90:grid r̲eshape ((
    ((g85:r̲ank (q, "q", 2, "q[i,d]")) g87:a̲t ((
      ((q, "q", 2, "d") g86:c̲heck (0 + g91:g̲rid 3)) + (2 s̲elect s̲hape q) × 
      ((q, "q", 1, "i") g86:c̲heck (0 + g91:g̲rid 1))))) × 
    ((g85:r̲ank (k, "k", 2, "k[j,d]")) g87:a̲t ((
      ((k, "k", 2, "d") g86:c̲heck (0 + g91:g̲rid 3)) + (2 s̲elect s̲hape k) × 
      ((k, "k", 1, "j") g86:c̲heck (0 + g91:g̲rid 2)))))))
} @) ÷ (2.0 ^ 0.5)))
} @))
ᵉe̲q< expands to
(({ @ →
  g94:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g93:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g92:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g95:grid ← 3 c̲at 3
  g96:g̲rid ← { k → g95:grid r̲eshape (o̲ffsets k s̲elect g95:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g95:grid) r̲eshape 0 }
  g95:grid r̲eshape (((
    ((g92:r̲ank (M, "M", 2, "M[i,j]")) g94:a̲t ((
      ((M, "M", 2, "j") g93:c̲heck (0 + g96:g̲rid 2)) + (2 s̲elect s̲hape M) × 
      ((M, "M", 1, "i") g93:c̲heck (0 + g96:g̲rid 1))))) × (e̲xp 
    ((g92:r̲ank (s, "s", 2, "s[i,j]")) g94:a̲t ((
      ((s, "s", 2, "j") g93:c̲heck (0 + g96:g̲rid 2)) + (2 s̲elect s̲hape s) × 
      ((s, "s", 1, "i") g93:c̲heck (0 + g96:g̲rid 1))))))) ÷ ({ @ →
  g97:grid ← 3 c̲at 3 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g98:g̲rid ← { k → g97:grid r̲eshape (o̲ffsets k s̲elect g97:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g97:grid) r̲eshape 0 }
  '+ r̲/₃ g97:grid r̲eshape ((
    ((g92:r̲ank (M, "M", 2, "M[i,m]")) g94:a̲t ((
      ((M, "M", 2, "m") g93:c̲heck (0 + g98:g̲rid 3)) + (2 s̲elect s̲hape M) × 
      ((M, "M", 1, "i") g93:c̲heck (0 + g98:g̲rid 1))))) × (e̲xp 
    ((g92:r̲ank (s, "s", 2, "s[i,m]")) g94:a̲t ((
      ((s, "s", 2, "m") g93:c̲heck (0 + g98:g̲rid 3)) + (2 s̲elect s̲hape s) × 
      ((s, "s", 1, "i") g93:c̲heck (0 + g98:g̲rid 1))))))))
} @)))
} @))
ᵉe̲q< expands to
(({ @ →
  g101:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
  g100:c̲heck ← { (A, name, k, from) e →
    n ← k s̲elect s̲hape A
    bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
    0 = t̲ally bad ? e
    "index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
  }
  g99:r̲ank ← { (A, name, q, ref) →
    (t̲ally s̲hape A) = q ? A
    "rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
  }
  g102:grid ← 3 c̲at 2
  g103:g̲rid ← { k → g102:grid r̲eshape (o̲ffsets k s̲elect g102:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g102:grid) r̲eshape 0 }
  g102:grid r̲eshape (({ @ →
  g104:grid ← 3 c̲at 2 c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..2" } (1 + 2 − 0))
  g105:g̲rid ← { k → g104:grid r̲eshape (o̲ffsets k s̲elect g104:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g104:grid) r̲eshape 0 }
  '+ r̲/₃ g104:grid r̲eshape ((
    ((g99:r̲ank (a, "a", 2, "a[i,j]")) g101:a̲t ((
      ((a, "a", 2, "j") g100:c̲heck (0 + g105:g̲rid 3)) + (2 s̲elect s̲hape a) × 
      ((a, "a", 1, "i") g100:c̲heck (0 + g105:g̲rid 1))))) × 
    ((g99:r̲ank (v, "v", 2, "v[j,d]")) g101:a̲t ((
      ((v, "v", 2, "d") g100:c̲heck (0 + g105:g̲rid 2)) + (2 s̲elect s̲hape v) × 
      ((v, "v", 1, "j") g100:c̲heck (0 + g105:g̲rid 3)))))))
} @))
} @))
Used in: c4b

headM : Char

value · line 115
headM ← @ ᶠˡᵒʷm̲ath< c#"
ᶠˡᵒʷm̲ath< expands to
((r#"\begin{aligned} "# c̲at r#"& "# c̲at r#"X"# c̲at r#" \qquad \triangleright \, x \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Q[i,d] | 3 2" ᵉt̲ex< "sum f:=0..3 | x[i,f] * Wq[f,d]") c̲at r#" \qquad \triangleright \, q \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("K[j,d] | 3 2" ᵉt̲ex< "sum f:=0..3 | x[j,f] * Wk[f,d]") c̲at r#" \qquad \triangleright \, k \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("V[j,d] | 3 2" ᵉt̲ex< "sum f:=0..3 | x[j,f] * Wv[f,d]") c̲at r#" \qquad \triangleright \, v \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("S[i,j] | 3 3" ᵉt̲ex< "(sum d:=0..1 | q[i,d] * k[j,d]) / sqrt(2.0)") c̲at r#" \qquad \triangleright \, s \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("A[i,j] | 3 3" ᵉt̲ex< "M[i,j] * exp(s[i,j]) / (sum m:=0..2 | M[i,m] * exp(s[i,m]))") c̲at r#" \qquad \triangleright \, a \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Y[i,d] | 3 2" ᵉt̲ex< "sum j:=0..2 | a[i,j] * v[j,d]") c̲at r#""# c̲at r#" \end{aligned}"#))
ᵉt̲ex< expands to
("Q_{i,d} = \\sum_{f=0}^{3} x_{i,f} \\, Wq_{f,d}")
ᵉt̲ex< expands to
("K_{j,d} = \\sum_{f=0}^{3} x_{j,f} \\, Wk_{f,d}")
ᵉt̲ex< expands to
("V_{j,d} = \\sum_{f=0}^{3} x_{j,f} \\, Wv_{f,d}")
ᵉt̲ex< expands to
("S_{i,j} = \\frac{\\sum_{d=0}^{1} q_{i,d} \\, k_{j,d}}{\\sqrt{2.0}}")
ᵉt̲ex< expands to
("A_{i,j} = \\frac{M_{i,j} \\, \\exp(s_{i,j})}{\\sum_{m=0}^{2} M_{i,m} \\, \\exp(s_{i,m})}")
ᵉt̲ex< expands to
("Y_{i,d} = \\sum_{j=0}^{2} a_{i,j} \\, v_{j,d}")

c4b : Char

value · line 124
c4b ← "the same as equations" ᵏt̲est head1 ᵏn̲ear head0