mirror of
https://github.com/wassname/Open-Assistant.git
synced 2026-08-11 11:13:12 +08:00
Merge branch 'main' into 24-web-deploy-aws
This commit is contained in:
Keith Stevens
@@ -9,6 +9,7 @@ services:
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: postgres
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "postgres"]
|
||||
interval: 2s
|
||||
|
||||
@@ -16,6 +16,7 @@ services:
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: ocgpt_website
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "postgres"]
|
||||
interval: 2s
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import numpy as np
|
||||
from scipy import log2
|
||||
from scipy.integrate import nquad
|
||||
from scipy.special import gammaln, psi
|
||||
from scipy.stats import dirichlet
|
||||
|
||||
|
||||
def make_range(*x):
|
||||
"""
|
||||
constructs leftover values for the simplex given the first k entries
|
||||
(0,x_k) = 1-(x_1+...+x_(k-1))
|
||||
"""
|
||||
return (0, max(0, 1 - sum(x)))
|
||||
|
||||
|
||||
def relative_entropy(p, q):
|
||||
"""
|
||||
relative entropy of the two given dirichlet distributions
|
||||
"""
|
||||
|
||||
def tmp(*x):
|
||||
"""
|
||||
First adds the last always forced entry to the input (the last x_last = 1-(x_1+...+x_(N)) )
|
||||
Then computes the relative entropy of posterior and prior for that datapoint
|
||||
"""
|
||||
x_new = np.append(x, 1 - sum(x))
|
||||
return p(x_new) * log2(p(x_new) / q(x_new))
|
||||
|
||||
return tmp
|
||||
|
||||
|
||||
def naive_monte_carlo_integral(fun, dim, samples=10_000_000):
|
||||
s = np.random.rand(dim - 1, samples)
|
||||
s = np.sort(np.concatenate((np.zeros((1, samples)), s, np.ones((1, samples)))), 0)
|
||||
# print(s)
|
||||
pos = np.diff(s, axis=0)
|
||||
# print(pos)
|
||||
res = fun(pos)
|
||||
return np.mean(res)
|
||||
|
||||
|
||||
def analytic_solution(a_post, a_prior):
|
||||
"""
|
||||
Analytic solution to the KL-divergence between two dirichlet distributions.
|
||||
Proof is in the Notion design doc.
|
||||
"""
|
||||
post_sum = np.sum(a_post)
|
||||
prior_sum = np.sum(a_prior)
|
||||
info = (
|
||||
gammaln(post_sum)
|
||||
- gammaln(prior_sum)
|
||||
- np.sum(gammaln(a_post))
|
||||
+ np.sum(gammaln(a_prior))
|
||||
- np.sum((a_post - a_prior) * (psi(a_post) - psi(post_sum)))
|
||||
)
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def infogain(a_post, a_prior):
|
||||
raise (
|
||||
"""For the love of good don't use this:
|
||||
it's insanely poorly conditioned, the worst numerical code I have ever written
|
||||
and it's slow as molasses. Use the analytic solution instead.
|
||||
|
||||
Maybe remove
|
||||
"""
|
||||
)
|
||||
args = len(a_prior)
|
||||
p = dirichlet(a_post).pdf
|
||||
q = dirichlet(a_prior).pdf
|
||||
(info, _) = nquad(relative_entropy(p, q), [make_range for _ in range(args - 1)], opts={"epsabs": 1e-8})
|
||||
# info = naive_monte_carlo_integral(relative_entropy(p,q), len(a_post))
|
||||
return info
|
||||
|
||||
|
||||
def uniform_expected_infogain(a_prior):
|
||||
mean_weight = dirichlet.mean(a_prior)
|
||||
print("weight", mean_weight)
|
||||
results = []
|
||||
for i, w in enumerate(mean_weight):
|
||||
a_post = a_prior.copy()
|
||||
a_post[i] = a_post[i] + 1
|
||||
results.append(w * analytic_solution(a_post, a_prior))
|
||||
return np.sum(results)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
a_prior = np.array([1, 1, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
|
||||
a_post = np.array([1, 1, 20, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
|
||||
|
||||
print("algebraic", analytic_solution(a_post, a_prior))
|
||||
# print("raw",infogain(a_post, a_prior))
|
||||
print("large infogain", uniform_expected_infogain(a_prior))
|
||||
print("post infogain", uniform_expected_infogain(a_post))
|
||||
# a_prior = np.array([1,1,1000])
|
||||
# print("small infogain",uniform_expected_infogain(a_prior))
|
||||
@@ -68,7 +68,7 @@ def get_winner(pairs):
|
||||
def get_ranking(pairs):
|
||||
"""
|
||||
Abuses concordance property to get a (not necessarily unqiue) ranking.
|
||||
The lack of uniqueness is due to the potential existance of multiple
|
||||
The lack of uniqueness is due to the potential existence of multiple
|
||||
equally ranked winners. We have to pick one, which is where
|
||||
the non-uniqueness comes from
|
||||
"""
|
||||
@@ -99,7 +99,7 @@ def ranked_pairs(ranks: List[List[int]]):
|
||||
tallies = tallies - tallies.T
|
||||
# print(tallies)
|
||||
# note: the resulting tally matrix should be skew-symmetric
|
||||
# order by strenght of victory (using tideman's original method, don't think it would make a difference for us)
|
||||
# order by strength of victory (using tideman's original method, don't think it would make a difference for us)
|
||||
sorted_majorities = []
|
||||
for i in range(len(ranks[0])):
|
||||
for j in range(len(ranks[i])):
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
from scipy.stats import kendalltau
|
||||
|
||||
|
||||
@dataclass
|
||||
class Voter:
|
||||
"""
|
||||
Represents a single voter.
|
||||
This tabulates the number of good votes, total votes,
|
||||
and points.
|
||||
We only put well-behaved people on the scoreboard and filter out the badly behaved ones
|
||||
"""
|
||||
|
||||
uid: Any
|
||||
num_votes: int
|
||||
num_good_votes: int
|
||||
num_prompts: int
|
||||
num_good_prompts: int
|
||||
num_rankings: int
|
||||
num_good_rankings: int
|
||||
|
||||
#####################
|
||||
voting_points: int
|
||||
prompt_points: int
|
||||
ranking_points: int
|
||||
|
||||
def voter_quality(self):
|
||||
return self.num_good_votes / self.num_votes
|
||||
|
||||
def rank_quality(self):
|
||||
return self.num_good_rankings / self.num_rankings
|
||||
|
||||
def prompt_quality(self):
|
||||
return self.num_good_prompts / self.num_prompts
|
||||
|
||||
def is_well_behaved(self, threshhold_vote, threshhold_prompt, threshhold_rank):
|
||||
return (
|
||||
self.voter_quality() > threshhold_vote
|
||||
and self.prompt_quality() > threshhold_prompt
|
||||
and self.rank_quality() > threshhold_rank
|
||||
)
|
||||
|
||||
def total_points(self, voting_weight, prompt_weight, ranking_weight):
|
||||
return (
|
||||
voting_weight * self.voting_points
|
||||
+ prompt_weight * self.prompt_points
|
||||
+ ranking_weight * self.ranking_points
|
||||
)
|
||||
|
||||
|
||||
def score_update_votes(new_vote: int, consensus: npt.ArrayLike, voter_data: Voter) -> Voter:
|
||||
"""
|
||||
This function returns the new "quality score" and points for a voter,
|
||||
after that voter cast a vote on a question.
|
||||
|
||||
This function is only to be run when archiving a question
|
||||
i.e. the question has had sufficiently many votes, or we cann't get more than "K" bits of information
|
||||
|
||||
The consensus is the array of all votes cast by all voters for that question
|
||||
We then update the voter data using the new information
|
||||
|
||||
Parameters:
|
||||
new_vote (int): the index of the vote cast by the voter
|
||||
consensus (ArrayLike): all votes cast for this question
|
||||
voter_data (Voter): a "Voter" object that represents the person casting the "new_vote"
|
||||
|
||||
Returns:
|
||||
updated_voter (Voter): the new "quality score" and points for the voter
|
||||
"""
|
||||
# produces the ranking of votes, e.g. for [100,300,200] it returns [0, 2, 1],
|
||||
# since 100 is the lowest, 300 the highest and 200 the middle value
|
||||
consensus_ranking = np.argsort(np.argsort(consensus))
|
||||
new_points = consensus_ranking[new_vote] + voter_data.voting_points
|
||||
|
||||
# we need to correct for 0 indexing, if you are closer to "right" than "wrong" of the conensus,
|
||||
# it's a good vote
|
||||
new_good_votes = int(consensus_ranking[new_vote] > (len(consensus) - 1) / 2) + voter_data.num_good_votes
|
||||
new_num_votes = voter_data.num_votes + 1
|
||||
return replace(voter_data, num_votes=new_num_votes, num_good_votes=new_good_votes, voting_points=new_points)
|
||||
|
||||
|
||||
def score_update_prompts(consensus: npt.ArrayLike, voter_data: Voter) -> Voter:
|
||||
"""
|
||||
This function returns the gain of points for a given prompt's votes
|
||||
|
||||
This function is only to be run when archiving a question
|
||||
i.e. the question has had sufficiently many votes, or we cann't get more than "K" bits of information
|
||||
|
||||
Parameters:
|
||||
consensus (ArrayLike): all votes cast for this question
|
||||
voter_data (Voter): a "Voter" object that represents the person that wrote the prompt
|
||||
|
||||
Returns:
|
||||
updated_voter (Voter): the new "quality score" and points for the voter
|
||||
"""
|
||||
# produces the ranking of votes, e.g. for [100,300,200] it returns [0, 2, 1],
|
||||
# since 100 is the lowest, 300 the highest and 200 the middle value
|
||||
consensus_ranking = np.arange(len(consensus)) - len(consensus) // 2 + 1
|
||||
delta_votes = np.sum(consensus_ranking * consensus)
|
||||
new_points = delta_votes + voter_data.prompt_points
|
||||
|
||||
# we need to correct for 0 indexing, if you are closer to "right" than "wrong" of the conensus,
|
||||
# it's a good vote
|
||||
new_good_prompts = int(delta_votes > 0) + voter_data.num_good_prompts
|
||||
new_num_prompts = voter_data.num_prompts + 1
|
||||
return replace(
|
||||
voter_data,
|
||||
num_prompts=new_num_prompts,
|
||||
num_good_prompts=new_good_prompts,
|
||||
prompt_points=new_points,
|
||||
)
|
||||
|
||||
|
||||
def score_update_ranking(user_ranking: npt.ArrayLike, consensus_ranking: npt.ArrayLike, voter_data: Voter) -> Voter:
|
||||
"""
|
||||
This function returns the gain of points for a given ranking's votes
|
||||
|
||||
This function is only to be run when archiving a question
|
||||
i.e. the question has had sufficiently many votes, or we cann't get more than "K" bits of information
|
||||
|
||||
we use the bubble-sort distance (or "kendall-tau" distance) to compare the two rankings
|
||||
we use this over spearman correlation since:
|
||||
"[Kendall's τ] approaches a normal distribution more rapidly than ρ, as N, the sample size, increases;
|
||||
and τ is also more tractable mathematically, particularly when ties are present"
|
||||
Gilpin, A. R. (1993). Table for conversion of Kendall's Tau to Spearman's
|
||||
Rho within the context measures of magnitude of effect for meta-analysis
|
||||
|
||||
Further in
|
||||
"research design and statistical analyses, second edition, 2003"
|
||||
the authors note that at least from an significance test POV they will yield the same p-values
|
||||
|
||||
Parameters:
|
||||
user_ranking (ArrayLike): ranking produced by the user
|
||||
consensus (ArrayLike): ranking produced after running the voting algorithm to merge into the consensus ranking
|
||||
voter_data (Voter): a "Voter" object that represents the person that wrote the prompt
|
||||
|
||||
Returns:
|
||||
updated_voter (Voter): the new "quality score" and points for the voter
|
||||
"""
|
||||
bubble_sort_distance, p_value = kendalltau(user_ranking, consensus_ranking)
|
||||
# normalize kendall-tau from [-1,1] into [0,1] range
|
||||
bubble_sort_distance = (1 + bubble_sort_distance) / 2
|
||||
new_points = bubble_sort_distance + voter_data.ranking_points
|
||||
new_good_rankings = int(bubble_sort_distance > 0.5) + voter_data.num_good_rankings
|
||||
new_num_rankings = voter_data.num_rankings + 1
|
||||
return replace(
|
||||
voter_data,
|
||||
num_rankings=new_num_rankings,
|
||||
num_good_rankings=new_good_rankings,
|
||||
ranking_points=new_points,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
demo_voter = Voter(
|
||||
"abc",
|
||||
num_votes=10,
|
||||
num_good_votes=2,
|
||||
num_prompts=10,
|
||||
num_good_prompts=2,
|
||||
num_rankings=10,
|
||||
num_good_rankings=2,
|
||||
voting_points=6,
|
||||
prompt_points=0,
|
||||
ranking_points=0,
|
||||
)
|
||||
new_vote = 3
|
||||
consensus = np.array([200, 300, 100, 500])
|
||||
print(demo_voter)
|
||||
print("best vote ", score_update_votes(new_vote, consensus, demo_voter))
|
||||
new_vote = 2
|
||||
print("worst vote ", score_update_votes(new_vote, consensus, demo_voter))
|
||||
new_vote = 1
|
||||
print("medium vote ", score_update_votes(new_vote, consensus, demo_voter))
|
||||
print("prompt writer", score_update_prompts(consensus, demo_voter))
|
||||
print("best rank ", score_update_ranking(np.array([0, 2, 1]), np.array([0, 2, 1]), demo_voter))
|
||||
print("medium rank ", score_update_ranking(np.array([2, 0, 1]), np.array([0, 2, 1]), demo_voter))
|
||||
print("worst rank ", score_update_ranking(np.array([1, 0, 2]), np.array([0, 2, 1]), demo_voter))
|
||||
Reference in New Issue
Block a user