mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-10 12:14:54 +08:00
Switch zone blobs to country-mean covariance ellipses, fit PCA on means
Per feedback the individual-respondent contours filled the frame (within >> between variance). Now each IW zone is a ~1.6-sigma covariance ellipse over its member country-mean dots, with an eigenvalue floor so 1-2 country zones get a visible blob instead of a dot/line (fixes big5 SG/PK orphans). PCA now fits on the country means M so the axes are between-country and zones separate. mfq2/big5/mfv read cleanly; humor still overlaps (real negative result: humor country profiles don't cluster the IW way). Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
@@ -749,3 +749,16 @@ over people; a person-level contour honestly shows the overlap the country-mean
|
||||
view hides. Secondary caveat: several mfq2 zones are single-country in the Atari
|
||||
19 (Confucian=Japan, Orthodox=Russia, Protestant=Switzerland), so those ellipses
|
||||
are one country's spread wearing a zone label.
|
||||
|
||||
Follow-up (same day, after eyeballing): the individual-respondent contour was the
|
||||
wrong call for the map -- it fills the frame and every zone overlaps, exactly
|
||||
because within >> between variance. Switched the zone blob to a ~1.6-sigma
|
||||
covariance ellipse over each zone's COUNTRY-MEAN dots (between-country spread) with
|
||||
an eigenvalue floor so a 1- or 2-country zone still gets a circle/ellipse instead
|
||||
of a dot/line. Also fit the ipsative PCA on the country means M (not the respondent
|
||||
cloud) so the axes are between-country. Result: mfq2/big5/mfv separate cleanly and
|
||||
Economist-like, every country is grouped (big5 SG/PK no longer orphaned). humor
|
||||
STILL overlaps heavily even on country means -- humor-style country profiles do not
|
||||
cluster the IW way on the top 2 ipsative PCs, a real negative result, not a plot
|
||||
bug. (Whether any linear axis separates humor zones is the LDA question, tested
|
||||
separately.)
|
||||
|
||||
@@ -153,14 +153,15 @@ def human_matrix(instr) -> tuple[list[str], np.ndarray]:
|
||||
return countries, _frac(raw, instr.human_scale_max)
|
||||
|
||||
|
||||
def human_haze(instr, n_per_country: int = 200, seed: int = 0) -> np.ndarray:
|
||||
"""Synthetic individual-respondent cloud (n x K, 0-1 fraction) for instruments that ship only
|
||||
society-level stats (big5/16pf/humor: no raw per-person data like mfq2's Atari file). For each
|
||||
(country, factor) we resample n Normal(mean, sd) draws from the published country mean+sd, so the
|
||||
cloud carries BOTH between-country (different means) and within-country (sd) human spread. Caveat:
|
||||
factors are drawn independently, so this marginal resample loses the cross-factor correlation a
|
||||
real respondent matrix has -- it is a backdrop envelope, not a covariance estimate, and is NOT
|
||||
used as the PCA basis (that stays the society means M)."""
|
||||
def human_haze(instr, n_per_country: int = 200, seed: int = 0) -> tuple[np.ndarray, list[str]]:
|
||||
"""Synthetic individual-respondent cloud (n x K, 0-1 fraction) + the country of each row, for
|
||||
instruments that ship only society-level stats (big5/16pf/humor: no raw per-person data like
|
||||
mfq2's Atari file). For each (country, factor) we resample n Normal(mean, sd) draws from the
|
||||
published country mean+sd, so the cloud carries BOTH between-country (different means) and
|
||||
within-country (sd) human spread. Caveat: factors are drawn independently, so this marginal
|
||||
resample loses the cross-factor correlation a real respondent matrix has -- it is a backdrop
|
||||
envelope, not a covariance estimate, and is NOT used as the PCA basis (that stays the society
|
||||
means M). The returned country-per-row list lets the map contour it by IW zone."""
|
||||
dims = instr.dimensions
|
||||
rng = np.random.default_rng(seed)
|
||||
stats: dict[tuple[str, str], tuple[float, float]] = {}
|
||||
@@ -168,11 +169,12 @@ def human_haze(instr, n_per_country: int = 200, seed: int = 0) -> np.ndarray:
|
||||
for r in csv.DictReader(fh):
|
||||
stats[(r["country"], r["foundation"])] = (float(r["mean"]), float(r["sd"]))
|
||||
countries = sorted({c for (c, _f) in stats})
|
||||
blocks = []
|
||||
blocks, row_country = [], []
|
||||
for c in countries:
|
||||
cols = [rng.normal(stats[(c, f)][0], stats[(c, f)][1], n_per_country) for f in dims]
|
||||
blocks.append(np.clip(np.stack(cols, axis=1), 1.0, instr.human_scale_max))
|
||||
return _frac(np.concatenate(blocks, axis=0), instr.human_scale_max)
|
||||
row_country.extend([c] * n_per_country)
|
||||
return _frac(np.concatenate(blocks, axis=0), instr.human_scale_max), row_country
|
||||
|
||||
|
||||
def human_strip(instr) -> dict[str, list[tuple[str, float]]]:
|
||||
@@ -276,23 +278,19 @@ def plot_ordinal(run_dir: Path, out: Path, name: str, vec_label: str, C: float,
|
||||
# fit the ipsative PCA on it (better-conditioned, the true envelope). Other instruments have no raw
|
||||
# per-person data, so scatter a marginal resample from each country's published mean+sd as the haze
|
||||
# while keeping the PCA basis on the society means M.
|
||||
# mfq2 has real per-respondent data -> draw a p90 respondent ELLIPSE per zone (grounded in
|
||||
# people) and drop the country-mean hull. Other instruments have only society means -> the
|
||||
# country-mean convex HULL is the best-available zone blob.
|
||||
_, emph = zones_for(countries)
|
||||
# Each IW zone is a covariance ellipse over its member COUNTRY-MEAN dots (drawn in maps). mfq2
|
||||
# scatters its real Atari respondents behind; the others scatter a per-country resample.
|
||||
zones, emph = zones_for(countries)
|
||||
if name == "mfq2":
|
||||
resp_countries, respondents = T.maps.respondent_profiles(dims, instr.scale_max)
|
||||
haze, zones = None, None
|
||||
respondent_zones = [_zone_of(c) for c in resp_countries]
|
||||
_, respondents = T.maps.respondent_profiles(dims, instr.scale_max)
|
||||
haze = None
|
||||
else:
|
||||
respondents, haze = None, human_haze(instr)
|
||||
zones, respondent_zones = zones_for(countries)[0], None
|
||||
respondents, (haze, _) = None, human_haze(instr)
|
||||
traj = {c: _frac(prof_c[c], instr.scale_max) for c in coh_cs}
|
||||
figm = T.maps.plot_ipsative_pca(instr, dims, countries, Mfrac,
|
||||
_frac(base, instr.scale_max), _frac(pos, instr.scale_max),
|
||||
_frac(neg, instr.scale_max), respondents=respondents, haze=haze,
|
||||
traj=traj, zones=zones, emphasize=emph,
|
||||
respondent_zones=respondent_zones, labels=labels)
|
||||
traj=traj, emphasize=emph, zones=zones, labels=labels)
|
||||
figm.axes[0].set_title(f"{instr.display}: humans vs LLMs steered for {vec_label}", fontsize=10)
|
||||
paths = [T.maps.save_both(figm, out / name, "map_pca_ipsative")]
|
||||
plt.close(figm)
|
||||
@@ -366,9 +364,9 @@ def plot_mfv_map(run_dir: Path, out: Path, vec_label: str, C: float, coh_cs: lis
|
||||
neg_c = min(c for c in coh_cs if c < 0.0)
|
||||
labels = ("base (c=0)", f"c={pos_c:+g}", f"c={neg_c:+g}")
|
||||
traj = {c: prof[c] for c in coh_cs}
|
||||
zones, emph = zones_for(countries)
|
||||
zones, emph = zones_for(countries) # MFV: 5 country dots, no cloud
|
||||
fig = T.maps.plot_ipsative_pca(_MFV_INSTR, founds, countries, M, prof[0.0], prof[pos_c], prof[neg_c],
|
||||
traj=traj, zones=zones, emphasize=emph, labels=labels)
|
||||
traj=traj, emphasize=emph, zones=zones, labels=labels)
|
||||
fig.axes[0].set_title(f"MFV vignettes: humans vs LLMs steered for {vec_label}", fontsize=10)
|
||||
path = T.maps.save_both(fig, out / "mfv", "map_pca_ipsative")
|
||||
plt.close(fig)
|
||||
|
||||
+28
-63
@@ -93,24 +93,6 @@ ZONE_COLORS = {
|
||||
}
|
||||
|
||||
|
||||
def convex_hull(pts: np.ndarray) -> np.ndarray:
|
||||
"""2D convex-hull vertices (CCW) via Andrew's monotone chain. Inline instead of scipy so the
|
||||
`maps` install extra stays matplotlib-only (scipy is dev-only). pts (n,2) -> polygon (m,2)."""
|
||||
P = sorted(map(tuple, pts.tolist()))
|
||||
if len(P) <= 2:
|
||||
return np.array(P, dtype=float)
|
||||
cross = lambda o, a, b: (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0])
|
||||
lower: list = []
|
||||
for p in P:
|
||||
while len(lower) >= 2 and cross(lower[-2], lower[-1], p) <= 0:
|
||||
lower.pop()
|
||||
lower.append(p)
|
||||
upper: list = []
|
||||
for p in reversed(P):
|
||||
while len(upper) >= 2 and cross(upper[-2], upper[-1], p) <= 0:
|
||||
upper.pop()
|
||||
upper.append(p)
|
||||
return np.array(lower[:-1] + upper[:-1], dtype=float)
|
||||
|
||||
|
||||
def save_both(fig, fig_dir: Path, stem: str, dpi: int = 200) -> Path:
|
||||
@@ -218,8 +200,8 @@ def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str],
|
||||
*, respondents: np.ndarray | None = None, haze: np.ndarray | None = None,
|
||||
traj: dict[float, np.ndarray] | None = None, traj_incoherent: set | None = None,
|
||||
boots: dict | None = None,
|
||||
zones: dict[str, list[str]] | None = None, emphasize: set[str] | None = None,
|
||||
respondent_zones: list[str] | None = None,
|
||||
emphasize: set[str] | None = None,
|
||||
zones: dict[str, list[str]] | None = None,
|
||||
labels: tuple[str, str, str] = ("baseline (c=0)", "honest (c=+2)", "dishonest (c=-2)")):
|
||||
"""Ipsative culture map. M is societies x K (0-1 fraction); base / pos / neg are the length-K
|
||||
fraction vectors for the base model and its two steer poles (or None). `labels` is the legend
|
||||
@@ -237,18 +219,19 @@ def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str],
|
||||
drawn hollow. `boots` optionally maps 'base'/'honest'/'dis' -> (n x K) bootstrap matrices.
|
||||
`zones` maps an Inglehart-Welzel zone name to the subset of `countries` (verbatim strings) in it;
|
||||
each zone with >=3 members gets a shaded convex hull (echoes the Economist WVS map's zone blobs),
|
||||
testing whether moral-foundation space recovers the WVS clusters. `emphasize` is a subset of
|
||||
`emphasize` is a subset of
|
||||
`countries` labelled bold-first so named outliers (China, US, Sweden...) always survive the
|
||||
label-collision drop. `respondent_zones` (length = rows of the projected cloud, i.e.
|
||||
`respondents` when `haze` is None) is each respondent's IW zone; each zone with enough respondents
|
||||
gets a p90 Gaussian ellipse of its cloud (the real-respondent analogue of a zone hull, mfq2 only).
|
||||
Returns the Figure."""
|
||||
label-collision drop. `zones` maps an IW zone name to its member `countries`; each becomes a
|
||||
covariance ellipse over that zone's COUNTRY-MEAN points (between-country spread, so zones stay
|
||||
separate -- contouring individual respondents instead gives huge overlap since within-culture
|
||||
variance dominates). An eigenvalue floor gives a 1- or 2-country zone a visible blob. The PCA is
|
||||
fit on the country means M; `respondents`/`haze` only scatter + set the crop. Returns the
|
||||
Figure."""
|
||||
try:
|
||||
import textalloc as ta
|
||||
except ImportError:
|
||||
ta = None
|
||||
fit_on = respondents if respondents is not None else M
|
||||
_, Vt, var, mu, Pc = ipsative_pca(fit_on) # signs already stabilized inside the helper
|
||||
_, Vt, var, mu, Pc = ipsative_pca(M) # fit on country means: between-country axes
|
||||
P = (M @ Pc - mu) @ Vt[:2].T
|
||||
cloud = haze if haze is not None else respondents # what we scatter + crop to (fit is separate)
|
||||
|
||||
@@ -259,50 +242,32 @@ def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str],
|
||||
fig, ax = plt.subplots(figsize=(8.5, 7.5))
|
||||
ax.set_facecolor("#faf8f2")
|
||||
ax.grid(True, color="#eceadf", lw=0.3, zorder=0)
|
||||
if cloud is not None: # grey haze = human respondents (rasterized; SVG-safe)
|
||||
if cloud is not None: # grey haze/respondents (rasterized; SVG-safe)
|
||||
Pi = (cloud @ Pc - mu) @ Vt[:2].T
|
||||
ax.scatter(Pi[:, 0], Pi[:, 1], s=4, c="#8f8a7e", alpha=0.14, edgecolors="none",
|
||||
zorder=1, rasterized=True)
|
||||
# Per-zone p90 respondent ellipse (mfq2 only): the real-respondent analogue of a zone hull.
|
||||
# A 90%-mass contour of the bivariate-Gaussian fit to each zone's projected cloud; the radius
|
||||
# is the chi-square(2 dof) 0.90 quantile, sqrt(4.605)=2.1459 (hardcoded to avoid a scipy dep).
|
||||
if respondent_zones is not None:
|
||||
assert len(respondent_zones) == Pi.shape[0], "respondent_zones must align with the cloud rows"
|
||||
zr = np.asarray(respondent_zones)
|
||||
for zname in dict.fromkeys(respondent_zones): # stable order, unique
|
||||
zp = Pi[zr == zname]
|
||||
if len(zp) < 30: # too few respondents -> unreliable contour
|
||||
continue
|
||||
cen = zp.mean(0)
|
||||
evals, evecs = np.linalg.eigh(np.cov(zp.T))
|
||||
ang = np.degrees(np.arctan2(evecs[1, -1], evecs[0, -1]))
|
||||
w, h = 2 * 2.1459 * np.sqrt(np.maximum(evals[::-1], 0))
|
||||
zcol = ZONE_COLORS.get(zname, "#888888")
|
||||
ax.add_patch(Ellipse(cen, w, h, angle=ang, facecolor="none",
|
||||
edgecolor=zcol, alpha=0.75, lw=1.4, ls="--", zorder=1.6))
|
||||
ax.text(cen[0], cen[1], zname, fontsize=8.5, color=zcol, ha="center",
|
||||
va="center", style="italic", fontweight="bold", zorder=2, alpha=0.9)
|
||||
# Inglehart-Welzel zone hulls: a shaded convex blob per zone with >=3 member societies, drawn
|
||||
# UNDER the society dots (zorder<3). The zone name sits at the hull centroid in grey, echoing the
|
||||
# Economist WVS map. A 2-member zone has no polygon, so it's shown as its connecting segment.
|
||||
# Per-zone blob: a ~1.6-sigma covariance ellipse over that zone's COUNTRY-MEAN points. Between-
|
||||
# country spread keeps the zones separate; an eigenvalue floor (a fraction of the overall P
|
||||
# spread) gives a 1-country zone a small circle and a 2-country zone real width instead of a line.
|
||||
if zones:
|
||||
cidx = {c: i for i, c in enumerate(countries)}
|
||||
floor = (0.07 * float(np.hypot(*(P.max(0) - P.min(0))))) ** 2
|
||||
for zname, members in zones.items():
|
||||
mi = [cidx[c] for c in members if c in cidx]
|
||||
if len(mi) < 2:
|
||||
zp = P[[cidx[c] for c in members if c in cidx]]
|
||||
if len(zp) == 0:
|
||||
continue
|
||||
zpts = P[mi]
|
||||
cen = zp.mean(0)
|
||||
cov = np.cov(zp.T) if len(zp) > 1 else np.zeros((2, 2))
|
||||
evals, evecs = np.linalg.eigh(cov)
|
||||
ang = np.degrees(np.arctan2(evecs[1, -1], evecs[0, -1]))
|
||||
w, h = 2 * 1.6 * np.sqrt(np.maximum(evals[::-1], floor))
|
||||
zcol = ZONE_COLORS.get(zname, "#888888")
|
||||
if len(mi) >= 3:
|
||||
hull = convex_hull(zpts)
|
||||
ax.add_patch(plt.Polygon(hull, closed=True, facecolor=zcol, edgecolor=zcol,
|
||||
alpha=0.13, lw=1.0, zorder=1.6))
|
||||
ax.plot(*np.vstack([hull, hull[:1]]).T, color=zcol, lw=1.0, alpha=0.45, zorder=1.7)
|
||||
else:
|
||||
ax.plot(zpts[:, 0], zpts[:, 1], color=zcol, lw=1.2, alpha=0.5, zorder=1.7)
|
||||
cx, cy = zpts.mean(0)
|
||||
ax.text(cx, cy, zname, fontsize=8.5, color="#6b6b6b", ha="center", va="center",
|
||||
style="italic", zorder=2, alpha=0.85)
|
||||
ax.add_patch(Ellipse(cen, w, h, angle=ang, facecolor=zcol, edgecolor=zcol,
|
||||
alpha=0.12, lw=1.0, zorder=1.5))
|
||||
ax.add_patch(Ellipse(cen, w, h, angle=ang, facecolor="none", edgecolor=zcol,
|
||||
alpha=0.7, lw=1.2, zorder=1.7))
|
||||
ax.text(cen[0], cen[1], zname, fontsize=8.5, color=zcol, ha="center",
|
||||
va="center", style="italic", fontweight="bold", zorder=2, alpha=0.9)
|
||||
ax.scatter(P[:, 0], P[:, 1], s=26, c=C_HUM, alpha=0.7, edgecolors="white", linewidths=0.5, zorder=3)
|
||||
# Society labels: each name/ISO code is pinned RIGHT NEXT to its dot (small fixed offset, no
|
||||
# leader line). A label is dropped if its box would collide with an already-placed one -- better an
|
||||
|
||||
Reference in New Issue
Block a user