diff --git a/scripts/wvs_direct_choice_production_pilot.py b/scripts/wvs_direct_choice_production_pilot.py index fc8f0a6..14f7982 100644 --- a/scripts/wvs_direct_choice_production_pilot.py +++ b/scripts/wvs_direct_choice_production_pilot.py @@ -85,7 +85,8 @@ def protocol_id(pilot_items: list[dict], request_plan: list[dict]) -> str: return direct_choice_protocol_identity( MODEL, pilot_items, samples_per_order=10, temperature=TEMPERATURE, max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT, reasoning=REASONING, - structured_output=STRUCTURED_OUTPUT, answer_instruction=ANSWER_INSTRUCTION, + structured_output=STRUCTURED_OUTPUT, prompt_instruction=PROMPT_INSTRUCTION, + answer_instruction=ANSWER_INSTRUCTION, rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan, ) @@ -169,7 +170,7 @@ def write_manifest(pilot_items: list[dict], model: dict, request_plan: list[dict "", "## Preregistered diagnostics", "", - "For every item, record the exact position-balance matrix, canonical-choice entropy normalized by log(n), and first-ten versus last-ten schedule-half total variation and modal sets. Report canonical/reversed direction distributions descriptively with their counts. Compare direct-choice distributions to Gemini's legacy dense-rated results descriptively only; never mix the two layers in coordinates, family summaries, or capability fits. Any failed request, missing parsed choice, or incomplete item exits nonzero and leaves no cache entry.", + "For every item, record the exact position-balance matrix, canonical-choice entropy normalized by log(n), and first-ten versus last-ten schedule-half total variation and modal sets. Also report the empirical selected-presented-position distribution, its normalized entropy and TV from uniform. TV >0.25 is a warning, not a hard exclusion; full schedule balance makes it interpretable, while n=3 is near-balanced. Report canonical/reversed direction distributions descriptively with their counts. Compare direct-choice distributions to Gemini's legacy dense-rated results descriptively only; never mix the two layers in coordinates, family summaries, or capability fits. Any failed request, missing parsed choice, or incomplete item exits nonzero and leaves no cache entry.", "", "## Spend check before dispatch", "", @@ -231,7 +232,8 @@ def main() -> None: MODEL, pilot_items, samples_per_order=10, temperature=TEMPERATURE, max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT, reasoning=REASONING, structured_output=STRUCTURED_OUTPUT, records_path=RECORDS_PATH, cache_path=CACHE_PATH, - answer_instruction=ANSWER_INSTRUCTION, rescue_instruction=RESCUE_INSTRUCTION, + prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION, + rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan, ) if result["cached"]: diff --git a/slop/audits/20260917_wvs_gemini37_direct_choice_task_1630_killed.md b/slop/audits/20260917_wvs_gemini37_direct_choice_task_1630_killed.md new file mode 100644 index 0000000..9318ee8 --- /dev/null +++ b/slop/audits/20260917_wvs_gemini37_direct_choice_task_1630_killed.md @@ -0,0 +1,22 @@ +# Task 1630 stopped before production result + +- Pueue task: 1630, `scripts/wvs_api/04_gemini37_direct_choice_production_pilot.sh` +- label: `why: test full behavior-values direct-choice readout after no-example control; resolve: audit balance, entropy, schedule halves before any other model` +- status: killed at 2026-09-17T11:28:16+08:00 after the protocol-metadata correction arrived +- partial ledger: `slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_production_requests.jsonl` + +## Observation + +The incomplete run uses the superseded protocol identity and has no parsed samples, item results, run-finished event, or cache entry. Its durable ledger records: + +- 1 `run_started` +- 11 `request_started` +- 10 `request_completed` +- 0 `answer_parsed` +- 0 `item_result` +- 0 `request_failed` +- observed completed-request cost: USD 0.0103275 + +The stopped task cannot supply a production panel and will not be merged with the corrected protocol's future records. The follow-up protocol records the behavioral-values prompt instruction explicitly, so cache identity metadata agrees with the rendered requests. + +-- PI[gpt-5.6-terra] diff --git a/slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_production_manifest.md b/slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_production_manifest.md index 60aed6b..7684b45 100644 --- a/slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_production_manifest.md +++ b/slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_production_manifest.md @@ -5,7 +5,7 @@ This preregistered direct-choice pilot is a separate legacy/proxy comparison lay ## Exact identity - model: `google/gemini-3.7-flash`; saved-catalog created UTC: 2026-08-13 -- protocol ID: `075bd0ef0c96982d69be237ef044671193c00d6ead75ef1963ff07f699ceab6d` +- protocol ID: `3e9c3d54727e46c92af49321604778d9bae85bd83a22e23e1793cbefd06f29e3` - cache: `slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_production_cache.json` - append-only request ledger: `slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_production_requests.jsonl` - 12 WVS items x 20 scheduled samples = 240 initial calls @@ -31,13 +31,13 @@ n=4 intentionally has 12 canonical and 8 reversed requests: exact equal position ## Preregistered diagnostics -For every item, record the exact position-balance matrix, canonical-choice entropy normalized by log(n), and first-ten versus last-ten schedule-half total variation and modal sets. Report canonical/reversed direction distributions descriptively with their counts. Compare direct-choice distributions to Gemini's legacy dense-rated results descriptively only; never mix the two layers in coordinates, family summaries, or capability fits. Any failed request, missing parsed choice, or incomplete item exits nonzero and leaves no cache entry. +For every item, record the exact position-balance matrix, canonical-choice entropy normalized by log(n), and first-ten versus last-ten schedule-half total variation and modal sets. Also report the empirical selected-presented-position distribution, its normalized entropy and TV from uniform. TV >0.25 is a warning, not a hard exclusion; full schedule balance makes it interpretable, while n=3 is near-balanced. Report canonical/reversed direction distributions descriptively with their counts. Compare direct-choice distributions to Gemini's legacy dense-rated results descriptively only; never mix the two layers in coordinates, family summaries, or capability fits. Any failed request, missing parsed choice, or incomplete item exits nonzero and leaves no cache entry. ## Spend check before dispatch - rated-ledger observed cost: USD 3.6235153224 -- prior direct-choice observed cost: USD 0.1069845000 -- cumulative observed cost: USD 3.7304998224 +- prior direct-choice observed cost: USD 0.1173120000 +- cumulative observed cost: USD 3.7408273224 - current output price: USD 3.75000000/M - 240 initial 1024-token completion-only ceiling: USD 0.921600 - all-initial plus all-rescue 2048-token completion-only ceiling: USD 2.764800; prompt tokens are additional diff --git a/slop/research/wvs/20260917_direct_choice/production_preflight_smoke.log b/slop/research/wvs/20260917_direct_choice/production_preflight_smoke.log index 67a44b7..f194962 100644 --- a/slop/research/wvs/20260917_direct_choice/production_preflight_smoke.log +++ b/slop/research/wvs/20260917_direct_choice/production_preflight_smoke.log @@ -1,4 +1,4 @@ smoke: 12 WVS items x 20 samples = 240 requests smoke: exact position balance for n=2,4,10; n=3 is nearest balance with max position difference 1 smoke: direction counts n=2/10 are 10/10, n=3 is 11/9, n=4 is 12/8 -smoke: distinct production protocol 075bd0ef0c96982d69be237ef044671193c00d6ead75ef1963ff07f699ceab6d +smoke: distinct production protocol 3e9c3d54727e46c92af49321604778d9bae85bd83a22e23e1793cbefd06f29e3 diff --git a/src/moralmaps/read_direct_choice.py b/src/moralmaps/read_direct_choice.py index e8c149f..34b94c0 100644 --- a/src/moralmaps/read_direct_choice.py +++ b/src/moralmaps/read_direct_choice.py @@ -49,6 +49,10 @@ def _force_choice(n: int) -> str: return f'Return ONLY {{"answer": }}. No explanation.' +_PROMPT_INSTRUCTION_OWN_VIEW = ( + "Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. " + "Answer immediately." +) _ANSWER_INSTRUCTION_WITH_EXAMPLE = ( 'Respond with ONLY a JSON object such as {{"answer": 0}}. The answer must be an integer from 0 through {max_index}.' ) @@ -135,6 +139,7 @@ def balanced_cyclic_plan(items: list[dict], total_samples: int, answer_instructi def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_per_order: int, temperature: float, max_tokens: int, concurrency: int, request_timeout: float, reasoning: dict, structured_output: bool, + prompt_instruction: str = _PROMPT_INSTRUCTION_OWN_VIEW, answer_instruction: str = _ANSWER_INSTRUCTION_WITH_EXAMPLE, rescue_instruction: str | None = None, plan_override: list[dict] | None = None) -> str: @@ -150,7 +155,7 @@ def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_pe "request_timeout": request_timeout, "reasoning": reasoning, "structured_output": structured_output, - "prompt_instruction": "Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. Answer immediately.", + "prompt_instruction": prompt_instruction, "response_schemas": {item["id"]: _choice_schema(item["n"]) for item in items}, "rescue_instructions": {item["id"]: rescue_instruction or _force_choice(item["n"]) for item in items}, "requests": plan, @@ -172,6 +177,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order temperature: float, max_tokens: int, concurrency: int, request_timeout: float, reasoning: dict, structured_output: bool, records_path: str | Path, cache_path: str | Path, + prompt_instruction: str = _PROMPT_INSTRUCTION_OWN_VIEW, answer_instruction: str = _ANSWER_INSTRUCTION_WITH_EXAMPLE, rescue_instruction: str | None = None, plan_override: list[dict] | None = None) -> dict: @@ -188,7 +194,8 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order protocol_id = direct_choice_protocol_identity( model, items, samples_per_order=samples_per_order, temperature=temperature, max_tokens=max_tokens, concurrency=concurrency, request_timeout=request_timeout, - reasoning=reasoning, structured_output=structured_output, answer_instruction=answer_instruction, + reasoning=reasoning, structured_output=structured_output, prompt_instruction=prompt_instruction, + answer_instruction=answer_instruction, rescue_instruction=rescue_instruction, plan_override=plan_override, ) cache_file = Path(cache_path) @@ -204,6 +211,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order "model": model, "samples_per_order": samples_per_order, "temperature": temperature, "max_tokens": max_tokens, "concurrency": concurrency, "request_timeout": request_timeout, "reasoning": reasoning, "structured_output": structured_output, + "prompt_instruction": prompt_instruction, } if plan_override is not None: settings.pop("samples_per_order")