sourcelibs/Statistics/src/Statistics.xtl
1⍝# Statistics: statistics -- median, quantiles, five-number summaries,
2⍝# z-scores, covariance, correlation, a least-squares line, binned
3⍝# counts; builds on the standard Stats.
4⍝# Import it with an alias of your choice: "sx:" u_se< "Statistics".
5⍝# Put libs/Statistics/src on XETAL_PATH ("just path"); the reference is libs/Statistics/docs.
6⍝# Names with l: are exported; those under h: are private to this file.
7⍝#
8⍝# X_eTaL's standard Stats has the mean, variance (of the population),
9⍝# standard deviation and range: import it beside this one
10⍝# ("s:" u_se< "Stats"). Results here are Floats.
11
12ˢ⁼u̲se< "Stats"
13
14⍝## Summaries
15
16⍝# The p quantile of sorted Floats x (0 <= p <= 1), interpolating
17⍝# between the two nearest items (R's type 7, Excel's PERCENTILE).
18ʰq̲1 ← { x p →
19 h ← p × f̲loat (t̲ally x) − 1 ⍝ 0-origin position
20 lo ← f̲loor h
21 hi ← (lo + 1) m̲in (t̲ally x) − 1
22 a ← f̲irst (1 + lo) s̲elect x
23 b ← f̲irst (1 + hi) s̲elect x
24 a + (h − f̲loat lo) × b − a
25}
26
27⍝# p q_uantile v: the p quantile of v for each p (0.5 is the median).
28ˡq̲uantile ← { p v →
29 0 = t̲ally v ? @ p̲anic< "the list is empty: a median or quantile needs at least one value"
30 '∨ r̲/ r̲avel (p < 0) ∨ p > 1 ? @ p̲anic< "a quantile is from 0 to 1, not {p}"
31 x ← s̲ort f̲loat v
32 '{ q → x ʰq̲1 q } e̲ach f̲loat p
33}
34
35⍝# m_edian v: the middle value (the mean of the two middle ones for an
36⍝# even count).
37ˡm̲edian ← { v → f̲irst 0.5 ˡq̲uantile v }
38
39⍝# f_ive v: the five-number summary: least, lower quartile, median,
40⍝# upper quartile, greatest.
41ˡf̲ive ← { v → 0.0 0.25 0.5 0.75 1.0 ˡq̲uantile v }
42
43⍝# z_scores v: how many standard deviations each item is from the mean.
44ˡz̲scores ← { v →
45 0.0 = ˢs̲d v ? @ p̲anic< "the values do not vary (their standard deviation is 0): z-scores are undefined"
46 ((f̲loat v) − ˢm̲ean v) ÷ ˢs̲d v
47}
48
49⍝## Relationships
50
51⍝# a c_ovariance b: the mean product of the deviations (population).
52ˡc̲ovariance ← { a b → ˢm̲ean ((f̲loat a) − ˢm̲ean a) × (f̲loat b) − ˢm̲ean b }
53
54⍝# a c_orrelation b: Pearson's correlation, from -1 to 1.
55ˡc̲orrelation ← { a b →
56 (0.0 = ˢs̲d a) ∨ 0.0 = ˢs̲d b ? @ p̲anic< "a list that does not vary (standard deviation 0) has no correlation"
57 (a ˡc̲ovariance b) ÷ (ˢs̲d a) × ˢs̲d b
58}
59
60⍝# x f_it y: the least-squares line through the points, intercept and
61⍝# slope (y is about intercept + slope * x).
62ˡf̲it ← { x y →
63 slope ← (x ˡc̲ovariance y) ÷ ˢv̲ariance x
64 ((ˢm̲ean y) − slope × ˢm̲ean x) c̲at slope
65}
66
67⍝## Binning
68
69⍝# n b_ins v: how many items of v fall in each of n bins of equal width
70⍝# from its least to its greatest (the greatest in the last bin).
71ˡb̲ins ← { n v →
72 lo ← 'm̲in r̲/ f̲loat v
73 step ← 0.000000000001 m̲ax (('m̲ax r̲/ f̲loat v) − lo) ÷ f̲loat n
74 bin ← 1 + (n − 1) m̲in f̲loor ((f̲loat v) − lo) ÷ step
75 '{ t̲ally w̲here bin = ⍵ } e̲ach r̲ange n
76}
p̲anic< expands to
(⎕P̲ANIC ("the values do not vary (their standard deviation is 0): z-scores are undefined"))