From be424c96a2d158f91cf4a7a57df45e2af73d8d33 Mon Sep 17 00:00:00 2001 From: Alexander Mattick Date: Sun, 25 Dec 2022 11:45:29 +0100 Subject: [PATCH 1/6] infogain analytic solution --- backend/postprocessing/infogain_selector.py | 98 +++++++++++++++++++++ backend/requirements.txt | 1 + 2 files changed, 99 insertions(+) create mode 100644 backend/postprocessing/infogain_selector.py diff --git a/backend/postprocessing/infogain_selector.py b/backend/postprocessing/infogain_selector.py new file mode 100644 index 00000000..51f60fa7 --- /dev/null +++ b/backend/postprocessing/infogain_selector.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +import numpy as np +from scipy import log2 +from scipy.integrate import nquad +from scipy.special import gammaln, psi +from scipy.stats import dirichlet + + +def make_range(*x): + """ + constructs leftover values for the simplex given the first k entries + (0,x_k) = 1-(x_1+...+x_(k-1)) + """ + return (0, max(0, 1 - sum(x))) + + +def relative_entropy(p, q): + """ + relative entropy of the two given dirichlet distributions + """ + + def tmp(*x): + """ + First adds the last always forced entry to the input (the last x_last = 1-(x_1+...+x_(N)) ) + Then computes the relative entropy of posterior and prior for that datapoint + """ + x_new = np.append(x, 1 - sum(x)) + return p(x_new) * log2(p(x_new) / q(x_new)) + + return tmp + + +def naive_monte_carlo_integral(fun, dim, samples=10_000_000): + s = np.random.rand(dim - 1, samples) + s = np.sort(np.concatenate((np.zeros((1, samples)), s, np.ones((1, samples)))), 0) + # print(s) + pos = np.diff(s, axis=0) + # print(pos) + res = fun(pos) + return np.mean(res) + + +def analytic_solution(a_post, a_prior): + """ + Analytic solution to the KL-divergence between two dirichlet distributions. + Proof is in the Notion design doc. + """ + post_sum = np.sum(a_post) + prior_sum = np.sum(a_prior) + info = ( + gammaln(post_sum) + - gammaln(prior_sum) + - np.sum(gammaln(a_post)) + + np.sum(gammaln(a_prior)) + - np.sum((a_post - a_prior) * (psi(a_post) - psi(post_sum))) + ) + + return info + + +def infogain(a_post, a_prior): + raise ( + """For the love of good don't use this: + it's insanely poorly conditioned, the worst numerical code I have ever written + and it's slow as molasses. Use the analytic solution instead. + + Maybe remove + """ + ) + args = len(a_prior) + p = dirichlet(a_post).pdf + q = dirichlet(a_prior).pdf + (info, _) = nquad(relative_entropy(p, q), [make_range for _ in range(args - 1)], opts={"epsabs": 1e-8}) + # info = naive_monte_carlo_integral(relative_entropy(p,q), len(a_post)) + return info + + +def uniform_expected_infogain(a_prior): + mean_weight = dirichlet.mean(a_prior) + print("weight", mean_weight) + results = [] + for i, w in enumerate(mean_weight): + a_post = a_prior.copy() + a_post[i] = a_post[i] + 1 + results.append(w * analytic_solution(a_post, a_prior)) + return np.sum(results) + + +if __name__ == "__main__": + a_prior = np.array([1, 1, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + a_post = np.array([1, 1, 20, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + print("algebraic", analytic_solution(a_post, a_prior)) + # print("raw",infogain(a_post, a_prior)) + print("large infogain", uniform_expected_infogain(a_prior)) + print("post infogain", uniform_expected_infogain(a_post)) + # a_prior = np.array([1,1,1000]) + # print("small infogain",uniform_expected_infogain(a_prior)) diff --git a/backend/requirements.txt b/backend/requirements.txt index b882d594..dd11aa18 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,6 +5,7 @@ numpy==1.22.4 psycopg2-binary==2.9.5 pydantic==1.9.1 python-dotenv==0.21.0 +scipy==1.8.1 SQLAlchemy==1.4.41 sqlmodel==0.0.8 starlette==0.22.0 From 76f7ed813b375df40b5eb2d06a998a71afff7308 Mon Sep 17 00:00:00 2001 From: Alexander Mattick Date: Sun, 25 Dec 2022 12:40:27 +0100 Subject: [PATCH 2/6] simple scoring system for scoreboard --- scripts/postprocessing/scoring.py | 65 +++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 scripts/postprocessing/scoring.py diff --git a/scripts/postprocessing/scoring.py b/scripts/postprocessing/scoring.py new file mode 100644 index 00000000..362a1c63 --- /dev/null +++ b/scripts/postprocessing/scoring.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +from dataclasses import dataclass +from typing import Any + +import numpy as np +import numpy.typing as npt + + +@dataclass +class Voter: + """ + Represents a single voter. + This tabulates the number of good votes, total votes, + and points. + We only put well-behaved people on the scoreboard and filter out the badly behaved ones + """ + + uid: Any + num_votes: int + num_good_votes: int + total_points: int + + def voter_quality(self): + return self.num_good_votes / self.num_votes + + def is_well_behaved(self, threshhold): + return self.voter_quality() > threshhold + + +def score_update(new_vote: int, consensus: npt.ArrayLike, voter_data: Voter) -> Voter: + """ + This function returns the new "quality score" and points for a voter. + I.e. a voter casts a vote on a question. + The consensus is the array of all votes cast by all voters for that question + We then update the voter data using the new information + + Parameters: + new_vote (int): the index of the vote cast by the voter + consensus (ArrayLike): all votes cast for this question + voter_data (Voter): a "Voter" object that represents the person casting the "new_vote" + + Returns: + updated_voter (Voter): the new "quality score" and points for the voter + """ + # produces the ranking of votes, e.g. for [100,300,200] it returns [0, 2, 1], + # since 100 is the lowest, 300 the highest and 200 the middle value + consensus_ranking = np.argsort(np.argsort(consensus)) + new_points = consensus_ranking[new_vote] + voter_data.total_points + # we need to correct for 0 indexing, if you are closer to "right" than "wrong" of the conensus, + # it's a good vote + new_good_votes = int(consensus_ranking[new_vote] > (len(consensus) - 1) / 2) + voter_data.num_good_votes + new_num_votes = voter_data.num_votes + 1 + return Voter(voter_data.uid, new_num_votes, new_good_votes, new_points) + + +if __name__ == "__main__": + demo_voter = Voter("abc", 10, 2, 6) + new_vote = 3 + consensus = np.array([200, 300, 100, 500]) + print(demo_voter) + print("best vote", score_update(new_vote, consensus, demo_voter)) + new_vote = 2 + print("worst vote", score_update(new_vote, consensus, demo_voter)) + new_vote = 1 + print("medium vote", score_update(new_vote, consensus, demo_voter)) From 58885f1e9b79cd6d1daf2115819773f6823751af Mon Sep 17 00:00:00 2001 From: Alexander Mattick Date: Sun, 25 Dec 2022 13:39:21 +0100 Subject: [PATCH 3/6] simple scoring system for prompts and ranks --- scripts/postprocessing/scoring.py | 132 +++++++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 11 deletions(-) diff --git a/scripts/postprocessing/scoring.py b/scripts/postprocessing/scoring.py index 362a1c63..47d85f3e 100644 --- a/scripts/postprocessing/scoring.py +++ b/scripts/postprocessing/scoring.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any import numpy as np import numpy.typing as npt +from scipy.stats import kendalltau @dataclass @@ -18,7 +19,15 @@ class Voter: uid: Any num_votes: int num_good_votes: int - total_points: int + num_prompts: int + num_good_prompts: int + num_rankings: int + num_good_rankings: int + + ##################### + voting_points: int + prompt_points: int + ranking_points: int def voter_quality(self): return self.num_good_votes / self.num_votes @@ -26,11 +35,22 @@ class Voter: def is_well_behaved(self, threshhold): return self.voter_quality() > threshhold + def total_points(self, voting_weight, prompt_weight, ranking_weight): + return ( + voting_weight * self.voting_points + + prompt_weight * self.prompt_points + + ranking_weight * self.ranking_points + ) -def score_update(new_vote: int, consensus: npt.ArrayLike, voter_data: Voter) -> Voter: + +def score_update_votes(new_vote: int, consensus: npt.ArrayLike, voter_data: Voter) -> Voter: """ - This function returns the new "quality score" and points for a voter. - I.e. a voter casts a vote on a question. + This function returns the new "quality score" and points for a voter, + after that voter cast a vote on a question. + + This function is only to be run when archiving a question + i.e. the question has had sufficiently many votes, or we cann't get more than "K" bits of information + The consensus is the array of all votes cast by all voters for that question We then update the voter data using the new information @@ -45,21 +65,111 @@ def score_update(new_vote: int, consensus: npt.ArrayLike, voter_data: Voter) -> # produces the ranking of votes, e.g. for [100,300,200] it returns [0, 2, 1], # since 100 is the lowest, 300 the highest and 200 the middle value consensus_ranking = np.argsort(np.argsort(consensus)) - new_points = consensus_ranking[new_vote] + voter_data.total_points + new_points = consensus_ranking[new_vote] + voter_data.voting_points + # we need to correct for 0 indexing, if you are closer to "right" than "wrong" of the conensus, # it's a good vote new_good_votes = int(consensus_ranking[new_vote] > (len(consensus) - 1) / 2) + voter_data.num_good_votes new_num_votes = voter_data.num_votes + 1 - return Voter(voter_data.uid, new_num_votes, new_good_votes, new_points) + return replace(voter_data, num_votes=new_num_votes, num_good_votes=new_good_votes, voting_points=new_points) + + +def score_update_prompts(consensus: npt.ArrayLike, voter_data: Voter) -> Voter: + """ + This function returns the gain of points for a given prompt's votes + + This function is only to be run when archiving a question + i.e. the question has had sufficiently many votes, or we cann't get more than "K" bits of information + + Parameters: + consensus (ArrayLike): all votes cast for this question + voter_data (Voter): a "Voter" object that represents the person that wrote the prompt + + Returns: + updated_voter (Voter): the new "quality score" and points for the voter + """ + # produces the ranking of votes, e.g. for [100,300,200] it returns [0, 2, 1], + # since 100 is the lowest, 300 the highest and 200 the middle value + consensus_ranking = np.argsort(np.argsort(consensus)) - len(consensus) // 2 + delta_votes = np.sum(consensus_ranking * consensus) + new_points = delta_votes + voter_data.prompt_points + + # we need to correct for 0 indexing, if you are closer to "right" than "wrong" of the conensus, + # it's a good vote + new_good_prompts = int(delta_votes > 0) + voter_data.num_good_prompts + new_num_prompts = voter_data.num_prompts + 1 + return replace( + voter_data, + num_prompts=new_num_prompts, + num_good_prompts=new_good_prompts, + prompt_points=new_points, + ) + + +def score_update_ranking(user_ranking: npt.ArrayLike, consensus_ranking: npt.ArrayLike, voter_data: Voter) -> Voter: + """ + This function returns the gain of points for a given ranking's votes + + This function is only to be run when archiving a question + i.e. the question has had sufficiently many votes, or we cann't get more than "K" bits of information + + we use the bubble-sort distance (or "kendall-tau" distance) to compare the two rankings + we use this over spearman correlation since: + "[Kendall's τ] approaches a normal distribution more rapidly than ρ, as N, the sample size, increases; + and τ is also more tractable mathematically, particularly when ties are present" + Gilpin, A. R. (1993). Table for conversion of Kendall's Tau to Spearman's + Rho within the context measures of magnitude of effect for meta-analysis + + Further in + "research design and statistical analyses, second edition, 2003" + the authors note that at least from an significance test POV they will yield the same p-values + + Parameters: + user_ranking (ArrayLike): ranking produced by the user + consensus (ArrayLike): ranking produced after running the voting algorithm to merge into the consensus ranking + voter_data (Voter): a "Voter" object that represents the person that wrote the prompt + + Returns: + updated_voter (Voter): the new "quality score" and points for the voter + """ + bubble_sort_distance, p_value = kendalltau(user_ranking, consensus_ranking) + # normalize kendall-tau from [-1,1] into [0,1] range + bubble_sort_distance = (1 + bubble_sort_distance) / 2 + new_points = bubble_sort_distance + voter_data.ranking_points + # it's a good ranking if the likelihood that the consensus ranking and the + # user ranking are related + new_good_rankings = int(p_value < 0.5) + voter_data.num_good_rankings + new_num_rankings = voter_data.num_rankings + 1 + return replace( + voter_data, + num_rankings=new_num_rankings, + num_good_rankings=new_good_rankings, + ranking_points=new_points, + ) if __name__ == "__main__": - demo_voter = Voter("abc", 10, 2, 6) + demo_voter = Voter( + "abc", + num_votes=10, + num_good_votes=2, + num_prompts=10, + num_good_prompts=2, + num_rankings=10, + num_good_rankings=2, + voting_points=6, + prompt_points=0, + ranking_points=0, + ) new_vote = 3 consensus = np.array([200, 300, 100, 500]) print(demo_voter) - print("best vote", score_update(new_vote, consensus, demo_voter)) + print("best vote ", score_update_votes(new_vote, consensus, demo_voter)) new_vote = 2 - print("worst vote", score_update(new_vote, consensus, demo_voter)) + print("worst vote ", score_update_votes(new_vote, consensus, demo_voter)) new_vote = 1 - print("medium vote", score_update(new_vote, consensus, demo_voter)) + print("medium vote ", score_update_votes(new_vote, consensus, demo_voter)) + print("prompt writer", score_update_prompts(consensus, demo_voter)) + print("best rank ", score_update_ranking(np.array([0, 2, 1]), np.array([0, 2, 1]), demo_voter)) + print("medium rank ", score_update_ranking(np.array([2, 0, 1]), np.array([0, 2, 1]), demo_voter)) + print("worst rank ", score_update_ranking(np.array([1, 0, 2]), np.array([0, 2, 1]), demo_voter)) From a1c2580027ddede98be42d10e69b7fb0cda61698 Mon Sep 17 00:00:00 2001 From: Alexander Mattick Date: Sun, 25 Dec 2022 13:42:59 +0100 Subject: [PATCH 4/6] added utility functions to dataclass --- scripts/postprocessing/scoring.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/postprocessing/scoring.py b/scripts/postprocessing/scoring.py index 47d85f3e..96af9a11 100644 --- a/scripts/postprocessing/scoring.py +++ b/scripts/postprocessing/scoring.py @@ -32,8 +32,18 @@ class Voter: def voter_quality(self): return self.num_good_votes / self.num_votes - def is_well_behaved(self, threshhold): - return self.voter_quality() > threshhold + def rank_quality(self): + return self.num_good_rankings / self.num_rankings + + def prompt_quality(self): + return self.num_good_prompts / self.num_prompts + + def is_well_behaved(self, threshhold_vote, threshhold_prompt, threshhold_rank): + return ( + self.voter_quality() > threshhold_vote + and self.prompt_quality() > threshhold_prompt + and self.rank_quality() > threshhold_rank + ) def total_points(self, voting_weight, prompt_weight, ranking_weight): return ( From d198eaf0458be6d5cb90f5575056642ff97aaedd Mon Sep 17 00:00:00 2001 From: Alexander Mattick Date: Sun, 25 Dec 2022 13:49:02 +0100 Subject: [PATCH 5/6] added fixed 'good prompt' definition for ranking --- scripts/postprocessing/scoring.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/postprocessing/scoring.py b/scripts/postprocessing/scoring.py index 96af9a11..fcfd49e7 100644 --- a/scripts/postprocessing/scoring.py +++ b/scripts/postprocessing/scoring.py @@ -146,9 +146,7 @@ def score_update_ranking(user_ranking: npt.ArrayLike, consensus_ranking: npt.Arr # normalize kendall-tau from [-1,1] into [0,1] range bubble_sort_distance = (1 + bubble_sort_distance) / 2 new_points = bubble_sort_distance + voter_data.ranking_points - # it's a good ranking if the likelihood that the consensus ranking and the - # user ranking are related - new_good_rankings = int(p_value < 0.5) + voter_data.num_good_rankings + new_good_rankings = int(bubble_sort_distance > 0.5) + voter_data.num_good_rankings new_num_rankings = voter_data.num_rankings + 1 return replace( voter_data, From a62db134142b2f9f62ba4797e8032bb03e013e17 Mon Sep 17 00:00:00 2001 From: Alexander Mattick Date: Sun, 25 Dec 2022 13:56:15 +0100 Subject: [PATCH 6/6] added fixed point definition for prompt --- scripts/postprocessing/scoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/postprocessing/scoring.py b/scripts/postprocessing/scoring.py index fcfd49e7..3c145b28 100644 --- a/scripts/postprocessing/scoring.py +++ b/scripts/postprocessing/scoring.py @@ -100,7 +100,7 @@ def score_update_prompts(consensus: npt.ArrayLike, voter_data: Voter) -> Voter: """ # produces the ranking of votes, e.g. for [100,300,200] it returns [0, 2, 1], # since 100 is the lowest, 300 the highest and 200 the middle value - consensus_ranking = np.argsort(np.argsort(consensus)) - len(consensus) // 2 + consensus_ranking = np.arange(len(consensus)) - len(consensus) // 2 + 1 delta_votes = np.sum(consensus_ranking * consensus) new_points = delta_votes + voter_data.prompt_points