sourcedemos/pipelines/diffusion.xtl
1⍝!/usr/bin/env xetal
2⍝│ Diffusion sampling four ways, a second Rosetta Stone: the two
3⍝│ equations of a latent diffusion sampler as a paper prints them, read
4⍝│ by X_eTaL-libraries' Equations; the sampler as a pipeline (Flow);
5⍝│ and in plain X_eTaL, as this repository's diffusion demo runs it.
6⍝│ The pipeline's expansion, the fourth way, is what `xetal expand
7⍝│ diffusion.xtl` shows. All give the same pictures.
8⍝│
9⍝│ The paper's two forms (bold z a latent, D the decoder, eps_theta the
10⍝│ denoiser conditioned on the text embedding of the prompt):
11⍝│ X0 = D(zT + sum_{t=T}^{1} [a_t z_t + b_t eps_theta(z_t, t, E_text(prompt)) + sigma_t eps_rand] - z_t)
12⍝│ X0 = D(z0) = D(zT + sum_{t=1}^{T} dz_t)
13⍝│ The first sum is a recurrence (its term at t uses the z_t the term
14⍝│ before made), so it is one step as the paper's summand, applied T
15⍝│ times (Equations' scan takes one seed for every element, not a
16⍝│ latent per element: ask E13); the second is a plain sum of the
17⍝│ steps' differences, and telescopes to the same z0. The networks are
18⍝│ called inside the equations by name (eps_theta and D), drawn as the
19⍝│ paper draws them.
20
21"e:" u̲se< "Equations"
22"flow:" u̲se< "Flow"
23"nn:" u̲se< "NN"
24"s:" u̲se< "Strings"
25"k:" u̲se< "Check"
26
27⍝━━ Data
28
29⍝│ A batch of N latents of K numbers, T steps; the schedule a_t, b_t
30⍝│ and sigma_t (the last step takes no noise, as DDIM), and the noise
31⍝│ eps_rand of every step, drawn once (a formula stands in for a
32⍝│ Gaussian draw, so the run is the same everywhere).
33N ← 2
34K ← 4
35T ← 3
36a ← 0.9 0.8 0.7
37b ← 0.1 0.2 0.3
38sigma ← 0.05 0.05 0.0
39noise ← (T c̲at N c̲at K) r̲eshape 0.5 × c̲os f̲loat r̲ange 24
40zT ← (N c̲at K) r̲eshape s̲in f̲loat r̲ange 8
41
42⍝│ The networks, stand-ins with made-up weights (the diffusion demo
43⍝│ trains real ones): the text encoder gives the prompt's embedding c;
44⍝│ the denoiser is a dense layer over each latent joined with its level
45⍝│ and c; the decoder a dense layer over the latent.
46c ← 0.5 -0.5
47We ← 8 4 r̲eshape 0.3 × s̲in f̲loat r̲ange 32
48Wd ← 5 4 r̲eshape 0.3 × c̲os f̲loat r̲ange 20
49ᵘe̲psTheta ← { (Z, t) → (Z c̲at₂ (N c̲at 3) r̲eshape (f̲loat t) c̲at c) ⁿⁿd̲ense We }
50ᵘd̲ecode ← { z → z ⁿⁿd̲ense Wd }
51⍝│ The paper's symbols for the networks in the equations' LaTeX.
52ᵘp̲aper ← { tex → r#"\operatorname{d_ecode}"# r#"\mathcal{D}"# ˢr̲eplace r#"\operatorname{e_psTheta}"# r#"\epsilon_\theta"# ˢr̲eplace tex }
53
54⍝━━ As a paper prints it
55
56⍝│ One step of the first sum, the paper's summand at level t: a_t z_t
57⍝│ + b_t eps_theta(z_t, t, c) + sigma_t eps_rand, read by Equations
58⍝│ over the batch (n) and the latent (k), the denoiser called inside
59⍝│ the equation (its argument, Z over the whole grid, is the latents);
60⍝│ the level goes down one.
61ᵘs̲tep ← { (Z, t) →
62 Zn ← ᵉe̲q< Zn_{n,k} = a_{t-1} \, Z_{n,k} + b_{t-1} \, \operatorname{e_psTheta}(Z_{n,k}, t) + sigma_{t-1} \, noise_{t-1,n,k}
63 (Zn, t − 1)
64}
65ᵘp̲aper "Zn[n,k] | N K" ᵉt̲ex< " | a[t-1] * Z[n,k] + b[t-1] * e_psTheta(Z[n,k], t) + sigma[t-1] * noise[t-1,n,k]"
66
67⍝│ The steps' differences dz_t = z_{t-1} - z_t, a row of N x K per
68⍝│ step, collected as the T steps run; then the second form, the sum
69⍝│ of them, and the decoder.
70ᵘc̲ollect ← { (Z, t, dz) →
71 (Zn, tn) ← ᵘs̲tep (Z, t)
72 (Zn, tn, dz c̲at (1 c̲at N c̲at K) r̲eshape Zn − Z)
73}
74(_, _, dz) ← T 'ᵘc̲ollect p̲ower (zT, T, (0 c̲at N c̲at K) r̲eshape 0.0)
75eq ← ᵉe̲q< X0_{n,k} = \operatorname{d_ecode}(zT_{n,k} + \left(\sum_{t=1}^{T} dz_{t-1,n,k}\right))
76ᵘp̲aper "X0[n,k] | N K" ᵉt̲ex< "d_ecode(zT[n,k] + (sum t:=1..T | dz[t-1,n,k]))"
77
78⍝│ The second form as a pipeline of equations: the latents of noise,
79⍝│ the steps' differences added, the decoder (a dense layer, its bias
80⍝│ Wd's last row); drawn as math a row a stage.
81eqpipe ← @ ᶠˡᵒʷp̲ipe< c#"
82 zT
83▷ "Z0[n,k] | N K" ᵉe̲q< "it[n,k] + (sum t:=1..T | dz[t-1,n,k])" ▹z0
84▷ "X0[n,j] | N K" ᵉe̲q< "(sum k:=0..K-1 | z0[n,k] * Wd[k,j]) + Wd[K,j]""#
85drawn ← @ ᶠˡᵒʷm̲ath< c#"
86 zT
87▷ "Z0[n,k] | N K" ᵉe̲q< "it[n,k] + (sum t:=1..T | dz[t-1,n,k])" ▹z0
88▷ "X0[n,j] | N K" ᵉe̲q< "(sum k:=0..K-1 | z0[n,k] * Wd[k,j]) + Wd[K,j]""#
89drawn
90
91⍝━━ As a pipeline
92
93⍝│ The latents of noise, the T steps (p_ower applies the step T times
94⍝│ to the latents and the level), the final latents labeled, the decoder.
95pipe ← @ ᶠˡᵒʷp̲ipe< c"
96 zT
97▷ T 'ᵘs̲tep p̲ower (it, T) ▹(z0, t0)
98▷ z0 ⁿⁿd̲ense Wd ▹X0 [N K]"
99
100⍝━━ Plain X_eTaL
101
102⍝│ The step without the macro: the schedule's values by level, the
103⍝│ noise's slab; then the sampler as the diffusion demo writes it.
104ᵘp̲lain ← { (Z, t) → (((t s̲elect a) × Z) + ((t s̲elect b) × ᵘe̲psTheta (Z, t)) + (t s̲elect sigma) × t s̲elect noise, t − 1) }
105(zp, _) ← T 'ᵘp̲lain p̲ower (zT, T)
106plain ← zp ⁿⁿd̲ense Wd
107
108⍝━━ The same
109
110r0 ← "the equations and the pipeline of equations" ᵏt̲est eq ᵏn̲ear eqpipe
111r1 ← "the equations and the pipeline" ᵏt̲est eq ᵏn̲ear pipe
112r2 ← "the pipeline and plain X_eTaL" ᵏt̲est pipe ᵏn̲ear plain
113ᵏr̲eport r0 ᵏa̲nd r1 ᵏa̲nd r2
114
115⍝│ The decoded pictures, a row per latent.
116plain
ᵉe̲q< expands to
(({ @ →
g3:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
g2:c̲heck ← { (A, name, k, from) e →
n ← k s̲elect s̲hape A
bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
0 = t̲ally bad ? e
"index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
}
g1:r̲ank ← { (A, name, q, ref) →
(t̲ally s̲hape A) = q ? A
"rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
}
g4:grid ← N c̲at K
g5:g̲rid ← { k → g4:grid r̲eshape (o̲ffsets k s̲elect g4:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g4:grid) r̲eshape 0 }
g4:grid r̲eshape (({ @ →
g6:grid ← N c̲at K
g7:g̲rid ← { k → g6:grid r̲eshape (o̲ffsets k s̲elect g6:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g6:grid) r̲eshape 0 }
g6:grid r̲eshape ((((
((g1:r̲ank (a, "a", 1, "a[t-1]")) g3:a̲t (
((a, "a", 1, "t-1") g2:c̲heck (t − 1)))) ×
((g1:r̲ank (Z, "Z", 2, "Z[n,k]")) g3:a̲t ((
((Z, "Z", 2, "k") g2:c̲heck (0 + g7:g̲rid 2)) + (2 s̲elect s̲hape Z) ×
((Z, "Z", 1, "n") g2:c̲heck (0 + g7:g̲rid 1)))))) + (
((g1:r̲ank (b, "b", 1, "b[t-1]")) g3:a̲t (
((b, "b", 1, "t-1") g2:c̲heck (t − 1)))) × (ᵘe̲psTheta(
((g1:r̲ank (Z, "Z", 2, "Z[n,k]")) g3:a̲t ((
((Z, "Z", 2, "k") g2:c̲heck (0 + g7:g̲rid 2)) + (2 s̲elect s̲hape Z) ×
((Z, "Z", 1, "n") g2:c̲heck (0 + g7:g̲rid 1))))), t)))) + (
((g1:r̲ank (sigma, "sigma", 1, "sigma[t-1]")) g3:a̲t (
((sigma, "sigma", 1, "t-1") g2:c̲heck (t − 1)))) ×
((g1:r̲ank (noise, "noise", 3, "noise[t-1,n,k]")) g3:a̲t ((
((noise, "noise", 3, "k") g2:c̲heck (0 + g7:g̲rid 2)) + (3 s̲elect s̲hape noise) × (
((noise, "noise", 2, "n") g2:c̲heck (0 + g7:g̲rid 1)) + (2 s̲elect s̲hape noise) ×
((noise, "noise", 1, "t-1") g2:c̲heck (t − 1)))))))))
} @))
} @))ᵉt̲ex< expands to
("Zn_{n,k} = a_{t-1} \\, Z_{n,k} + b_{t-1} \\, \\operatorname{e_psTheta}(Z_{n,k}, t) + sigma_{t-1} \\, noise_{t-1,n,k}")ᵉe̲q< expands to
(({ @ →
g10:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
g9:c̲heck ← { (A, name, k, from) e →
n ← k s̲elect s̲hape A
bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
0 = t̲ally bad ? e
"index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
}
g8:r̲ank ← { (A, name, q, ref) →
(t̲ally s̲hape A) = q ? A
"rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
}
g11:grid ← N c̲at K
g12:g̲rid ← { k → g11:grid r̲eshape (o̲ffsets k s̲elect g11:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g11:grid) r̲eshape 0 }
g11:grid r̲eshape ((ᵘd̲ecode((
((g8:r̲ank (zT, "zT", 2, "zT[n,k]")) g10:a̲t ((
((zT, "zT", 2, "k") g9:c̲heck (0 + g12:g̲rid 2)) + (2 s̲elect s̲hape zT) ×
((zT, "zT", 1, "n") g9:c̲heck (0 + g12:g̲rid 1))))) + ({ @ →
g13:grid ← N c̲at K c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 1..T" } (1 + T − 1))
g14:g̲rid ← { k → g13:grid r̲eshape (o̲ffsets k s̲elect g13:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g13:grid) r̲eshape 0 }
'+ r̲/₃ g13:grid r̲eshape (
((g8:r̲ank (dz, "dz", 3, "dz[t-1,n,k]")) g10:a̲t ((
((dz, "dz", 3, "k") g9:c̲heck (0 + g14:g̲rid 2)) + (3 s̲elect s̲hape dz) × (
((dz, "dz", 2, "n") g9:c̲heck (0 + g14:g̲rid 1)) + (2 s̲elect s̲hape dz) ×
((dz, "dz", 1, "t-1") g9:c̲heck ((1 + g14:g̲rid 3) − 1)))))))
} @)))))
} @))ᵉt̲ex< expands to
("X0_{n,k} = \\operatorname{d_ecode}(zT_{n,k} + \\left(\\sum_{t=1}^{T} dz_{t-1,n,k}\\right))")ᶠˡᵒʷp̲ipe< expands to
({ @ →
g15:stage ← zT
z0 ← { @ →
it ← g15:stage
"Z0[n,k] | N K" ᵉe̲q< "it[n,k] + (sum t:=1..T | dz[t-1,n,k])"
} @
g16:stage ← "X0[n,j] | N K" ᵉe̲q< "(sum k:=0..K-1 | z0[n,k] * Wd[k,j]) + Wd[K,j]"
g16:stage
} @)ᵉe̲q< expands to
({ @ →
g19:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
g18:c̲heck ← { (A, name, k, from) e →
n ← k s̲elect s̲hape A
bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
0 = t̲ally bad ? e
"index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
}
g17:r̲ank ← { (A, name, q, ref) →
(t̲ally s̲hape A) = q ? A
"rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
}
g20:grid ← N c̲at K
g21:g̲rid ← { k → g20:grid r̲eshape (o̲ffsets k s̲elect g20:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g20:grid) r̲eshape 0 }
g20:grid r̲eshape ((
((g17:r̲ank (it, "it", 2, "it[n,k]")) g19:a̲t ((
((it, "it", 2, "k") g18:c̲heck (0 + g21:g̲rid 2)) + (2 s̲elect s̲hape it) ×
((it, "it", 1, "n") g18:c̲heck (0 + g21:g̲rid 1))))) + ({ @ →
g22:grid ← N c̲at K c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 1..T" } (1 + T − 1))
g23:g̲rid ← { k → g22:grid r̲eshape (o̲ffsets k s̲elect g22:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g22:grid) r̲eshape 0 }
'+ r̲/₃ g22:grid r̲eshape (
((g17:r̲ank (dz, "dz", 3, "dz[t-1,n,k]")) g19:a̲t ((
((dz, "dz", 3, "k") g18:c̲heck (0 + g23:g̲rid 2)) + (3 s̲elect s̲hape dz) × (
((dz, "dz", 2, "n") g18:c̲heck (0 + g23:g̲rid 1)) + (2 s̲elect s̲hape dz) ×
((dz, "dz", 1, "t-1") g18:c̲heck ((1 + g23:g̲rid 3) − 1)))))))
} @)))
} @)ᵉe̲q< expands to
(({ @ →
g26:a̲t ← { A i → (1 + i) s̲elect r̲avel A }
g25:c̲heck ← { (A, name, k, from) e →
n ← k s̲elect s̲hape A
bad ← w̲here n̲ot r̲avel (0 ≤ e) ∧ e < n
0 = t̲ally bad ? e
"index" ⎕S̲IGNAL name c̲at " axis " c̲at (f̲ormat k) c̲at ": index " c̲at (f̲ormat (f̲irst bad) s̲elect r̲avel e) c̲at " is outside 0.." c̲at (f̲ormat n − 1) c̲at " (from " c̲at from c̲at ")"
}
g24:r̲ank ← { (A, name, q, ref) →
(t̲ally s̲hape A) = q ? A
"rank" ⎕S̲IGNAL ref c̲at " indexes " c̲at (f̲ormat q) c̲at " axes, but " c̲at name c̲at " has rank " c̲at f̲ormat t̲ally s̲hape A
}
g27:grid ← N c̲at K
g28:g̲rid ← { k → g27:grid r̲eshape (o̲ffsets k s̲elect g27:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g27:grid) r̲eshape 0 }
g27:grid r̲eshape ((({ @ →
g29:grid ← N c̲at K c̲at ({ v → (0 ≤ v) ? v◆ "bad-equation" ⎕S̲IGNAL "sum's range runs low..high: 0..K-1" } (1 + (K − 1) − 0))
g30:g̲rid ← { k → g29:grid r̲eshape (o̲ffsets k s̲elect g29:grid) 'l̲eft t̲able ('× r̲/ k d̲rop g29:grid) r̲eshape 0 }
'+ r̲/₃ g29:grid r̲eshape ((
((g24:r̲ank (z0, "z0", 2, "z0[n,k]")) g26:a̲t ((
((z0, "z0", 2, "k") g25:c̲heck (0 + g30:g̲rid 3)) + (2 s̲elect s̲hape z0) ×
((z0, "z0", 1, "n") g25:c̲heck (0 + g30:g̲rid 1))))) ×
((g24:r̲ank (Wd, "Wd", 2, "Wd[k,j]")) g26:a̲t ((
((Wd, "Wd", 2, "j") g25:c̲heck (0 + g30:g̲rid 2)) + (2 s̲elect s̲hape Wd) ×
((Wd, "Wd", 1, "k") g25:c̲heck (0 + g30:g̲rid 3)))))))
} @) +
((g24:r̲ank (Wd, "Wd", 2, "Wd[K,j]")) g26:a̲t ((
((Wd, "Wd", 2, "j") g25:c̲heck (0 + g28:g̲rid 2)) + (2 s̲elect s̲hape Wd) ×
((Wd, "Wd", 1, "K") g25:c̲heck K))))))
} @))ᶠˡᵒʷm̲ath< expands to
((r#"\begin{aligned} "# c̲at r#"& "# c̲at r#"zT"# c̲at r#" \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("Z0[n,k] | N K" ᵉt̲ex< "zT[n,k] + (sum t:=1..T | dz[t-1,n,k])") c̲at r#" \qquad \triangleright \, z0 \\ "# c̲at r#"\longrightarrow \; & "# c̲at ("X0[n,j] | N K" ᵉt̲ex< "(sum k:=0..K-1 | z0[n,k] * Wd[k,j]) + Wd[K,j]") c̲at r#""# c̲at r#" \end{aligned}"#))
ᵉt̲ex< expands to
("Z0_{n,k} = zT_{n,k} + \\left(\\sum_{t=1}^{T} dz_{t-1,n,k}\\right)")ᵉt̲ex< expands to
("X0_{n,j} = \\left(\\sum_{k=0}^{K - 1} z0_{n,k} \\, Wd_{k,j}\\right) + Wd_{K,j}")ᶠˡᵒʷp̲ipe< expands to
({ @ →
g31:stage ← zT
(z0, t0) ← { @ →
it ← g31:stage
T 'ᵘs̲tep p̲ower (it, T)
} @
X0 ← z0 ⁿⁿd̲ense Wd
_ ← { @ →
((s̲hape X0) m̲atch r̲avel N c̲at K) ? @
"shape" ⎕S̲IGNAL "stage 3 (X0): shape " c̲at (f̲ormat s̲hape X0) c̲at ", expected " c̲at f̲ormat r̲avel N c̲at K
} @
X0
} @)