From 6d5030d668a9b826122bbd9bc96d5b4ec1fa0895 Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Wed, 15 Mar 2023 06:03:45 -0400 Subject: [PATCH] Made various unmemorable changes --- src/Embeddings Search/testing.ipynb | 2416 ++++++++++++++++++++++----- 1 file changed, 2029 insertions(+), 387 deletions(-) diff --git a/src/Embeddings Search/testing.ipynb b/src/Embeddings Search/testing.ipynb index e27f945..473ba99 100644 --- a/src/Embeddings Search/testing.ipynb +++ b/src/Embeddings Search/testing.ipynb @@ -149,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -181,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -208,7 +208,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -259,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -269,7 +269,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -360,7 +360,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -375,7 +375,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -443,10 +443,11 @@ " def get_alignment_texts(self):\n", " with jsonlines.open(self.path, \"r\") as reader:\n", " for entry in reader:\n", - " random_number = random.random()\n", - " if random_number > self.fraction_of_articles_to_use: continue\n", " try:\n", " if 'source' not in entry: raise MissingDataException(\"Entry has no source.\")\n", + " random_number = random.random()\n", + " if random_number > self.fraction_of_articles_to_use:\n", + " continue\n", " \n", " if self.sources is None:\n", " if entry['source'] not in self.num_articles: self.num_articles[entry['source']] = 1\n", @@ -462,14 +463,18 @@ " \n", " # Get text\n", " if 'text' in entry and entry['text']: text = entry['text']\n", - " else: raise MissingDataException(f\"Entry has no text.\")\n", + " else:\n", + " raise MissingDataException(f\"Entry has no text.\")\n", " \n", " # Get title\n", " if 'title' in entry and 'book_title' in entry and entry['title']: title = entry['title']\n", - " elif 'book_title' in entry and 'title' not in entry and entry['book_title']: title = entry['book_title']\n", - " elif 'title' in entry and entry['title']: title = entry['title']\n", + " elif 'book_title' in entry and 'title' not in entry and entry['book_title']: \n", + " title = entry['book_title']\n", + " if title[-1] == '\\n': title = title[:-1]\n", + " elif 'title' in entry and entry['title']: \n", + " title = entry['title']\n", + " if title[-1] == '\\n': title = title[:-1]\n", " else: title = None\n", - " if title[-1] == '\\n': title = title[:-1]\n", " \n", " # Get author\n", " if 'author' in entry and 'authors' in entry and entry['author']: author = entry['author']\n", @@ -523,7 +528,7 @@ "\n", " def get_embeddings(self):\n", " # Get an embedding for each text, with retries if necessary\n", - " @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(10))\n", + " # @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(10))\n", " def get_embedding(text: str, delay_in_seconds: float = 0) -> np.ndarray:\n", " time.sleep(delay_in_seconds)\n", " result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n", @@ -553,55 +558,580 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ - "dataset = Dataset(path=PATH_TO_DATA, sources=None, rate_limit_per_minute=3500, block_min_max_size = [1200, 1500], fraction_of_articles_to_use=1/1000)\n", + "# List of possible sources:\n", + "all_sources = [\"https://aipulse.org\", \"ebook\", \"https://qualiacomputing.com\", \"alignment forum\", \"lesswrong\", \"manual\", \"arxiv\", \"https://deepmindsafetyresearch.medium.com\", \"waitbutwhy.com\", \"GitHub\", \"https://aiimpacts.org\", \"arbital.com\", \"carado.moe\", \"nonarxiv_papers\", \"https://vkrakovna.wordpress.com\", \"https://jsteinhardt.wordpress.com\", \"audio-transcripts\", \"https://intelligence.org\", \"youtube\", \"reports\", \"https://aisafety.camp\", \"curriculum\", \"https://www.yudkowsky.net\", \"distill\"]\n", + "\n", + "sources = [\"alignment forum\", #\"lesswrong\", \n", + " \"manual\", \"waitbutwhy.com\", \"https://aiimpacts.org\", \"arbital.com\", \"https://intelligence.org\", \"reports\", \"https://aisafety.camp\", \"curriculum\", \"https://www.yudkowsky.net\", \"distill\"]\n", + "\n", + "\n", + "dataset = Dataset(path=PATH_TO_DATA, sources=sources, rate_limit_per_minute=3500, block_min_max_size = [1200, 1500], fraction_of_articles_to_use=1/20)\n", "dataset.get_alignment_texts()" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Title: A study in Science on memory conformity\n", - "Author: dvasya\n", - "Date published: 2011-07-15\n", - "URL: https://www.lesswrong.com/posts/fQE3cP9NjS5sLm8wr/a-study-in-science-on-memory-conformity\n", - "Tags: Conformity Bias\n", - "Text: I believe this may be a good addition to the cognitive bias literature:\n", + "Title: AXRP Episode 13 - First Principles of AGI Safety with Richard Ngo\n", + "Author: DanielFilan\n", + "Date published: 2022-03-31\n", + "URL: https://www.lesswrong.com/posts/tEf8fEFCkFtPyg9pm/axrp-episode-13-first-principles-of-agi-safety-with-richard\n", + "Tags: AI/Audio/AXRP/Interviews/AI Risk/Existential Risk\n", + "Text: Link post\n", + "Contents\n", + " - The nature of intelligence and AGI \n", + " - The nature of intelligence \n", + " - AGI: what and how \n", + " - Single vs collective AI minds \n", + " - AGI in practice \n", + " - Impact \n", + " - Timing \n", + " - Creation \n", + " - Risks and benefits \n", + " - Making AGI safe \n", + " - Robustness of the agency abstraction \n", + " - Pivotal acts \n", + " - AGI safety concepts \n", + " - Alignment \n", + " - Transparency \n", + " - Cooperation \n", + " - Optima and selection pressures \n", + " - The AI alignment research community \n", + " - Updates from Yudkowsky conversation \n", + " - Corrections to the community \n", + " - Why others don’t join \n", + " - Richard Ngo as a researcher \n", + " - The world approaching AGI \n", + " - Following Richard’s work \n", + "Google Podcasts link\n", + "\n", + "This podcast is called AXRP, pronounced axe-urp and short for the AI X-risk Research Podcast. Here, I (Daniel Filan) have conversations with researchers about their papers. We discuss the paper and hopefully get a sense of why it’s been written and how it might reduce the risk of artificial intelligence causing an existential catastrophe: that is, permanently and drastically curtailing humanity’s future potential.\n", + "\n", + "How should we think about artificial general intelligence (AGI), and the risks it might pose? What constraints exist on technical solutions to the problem of aligning superhuman AI systems with human intentions? In this episode, I talk to Richard Ngo about his report analyzing AGI safety from first principles, and recent conversations he had with Eliezer Yudkowsky about the difficulty of AI alignment.\n", + "\n", + "Topics we discuss:\n", "\n", "\n", - "# > Following the Crowd: Brain Substrates of Long-Term Memory Conformity\n", + " - The nature of intelligence and AGI\n", + "\n", + " - The nature of intelligence\n", + "\n", + " - AGI: what and how\n", + "\n", + " - Single vs collective AI minds\n", "\n", "\n", - " - Micah Edelson1,*, \n", "\n", - " - Tali Sharot2, \n", + " - AGI in practice\n", "\n", - " - Raymond J. Dolan2, \n", + " - Impact\n", "\n", - " - Yadin Dudai1\n", + " - Timing\n", "\n", - "1Department of Neurobiology, Weizmann Institute of Science, Israel.\n", + " - Creation\n", + "\n", + " - Risks and benefits\n", "\n", "\n", - " - 2Wellcome Trust Centre for Neuroimaging, Institute of Neurology, University College London, London, UK.\n", "\n", - "## ABSTRACT\n", + " - Making AGI safe\n", "\n", - "Human memory is strikingly susceptible to social influences, yet we know little about the underlying mechanisms. We examined how socially induced memory errors are generated in the brain by studying the memory of individuals exposed to recollections of others. Participants exhibited a strong tendency to conform to erroneous recollections of the group, producing both long-lasting and temporary errors, even when their initial memory was strong and accurate. Functional brain imaging revealed that social influence modified the neuronal representation of memory. Specifically, a particular brain signature of enhanced amygdala activity and enhanced amygdala-hippocampus connectivity predicted long-lasting but not temporary memory alterations. Our findings reveal how social manipulation can alter memory and extend the known functions of the amygdala to encompass socially mediated memory distortions.\n", + " - Robustness of the agency abstraction\n", + "\n", + " - Pivotal acts\n", "\n", "\n", - "http://​​www.sciencemag.org/​​content/​​333/​​6038/​​108.full\n", "\n", - "http://​​ifile.it/​​v76wsi5\n", + " - AGI safety concepts\n", + "\n", + " - Alignment\n", + "\n", + " - Transparency\n", + "\n", + " - Cooperation\n", + "\n", + "\n", + "\n", + " - Optima and selection pressures\n", + "\n", + " - The AI alignment research community\n", + "\n", + " - Updates from Yudkowsky conversation\n", + "\n", + " - Corrections to the community\n", + "\n", + " - Why others don’t join\n", + "\n", + "\n", + "\n", + " - Richard Ngo as a researcher\n", + "\n", + " - The world approaching AGI\n", + "\n", + " - Following Richard’s work\n", + "\n", + "**Daniel Filan:**\n", + "Hello, everybody. Today, I’ll be speaking with Richard Ngo. Richard is a researcher at OpenAI, where he works on AI governance and forecasting. He also was a research engineer at DeepMind, and designed the course \"AGI Safety Fundamentals\". We’ll be discussing his report, AGI Safety from First Principles, as well as his debate with Eliezer Yudkowsky about the difficulty of AI alignment. For links to what we’re discussing, you can check the description of this episode, and you can read the transcripts at axrp.net. Well, Richard, welcome to the show.\n", + "\n", + "**Richard Ngo:**\n", + "Thanks so much for having me.\n", + "\n", + "## The nature of intelligence and AGI \n", + "\n", + "**Daniel Filan:**\n", + "You wrote this report, AGI Safety from First Principles. What is AGI? What do you mean by that term?\n", + "\n", + "**Richard Ngo:**\n", + "Fundamentally, it’s AI that can perform as well as humans, or comparably to humans on a wide range of tasks, because it’s got this thing called general intelligence that humans also have. And I think this is not a very precise definition. I think we just don’t understand what general intelligence really is, like how to specifically pin down this concept, but it does seem like there’s something special about humans that allows us to do all the things we do. And that’s the thing that we expect AGI to have.\n", + "\n", + "### The nature of intelligence \n", + "\n", + "**Daniel Filan:**\n", + "And so I shouldn’t be imagining that there’s some really technical definition of general or intelligence that you’re using here?\n", + "\n", + "**Richard Ngo:**\n", + "No, I think it’s much more like a pointer to a, let’s say, pre-paradigmatic concept. We kind of know the shape of it, but we don’t understand in mechanistic detail how general intelligence works. It’s got these components like: of course, you need memory, of course you need some kind of planning, but exactly how to characterize the ways in which combinations of these count as general intelligence is something we don’t have yet.\n", + "\n", + "**Daniel Filan:**\n", + "So when you’re talking about AGI, often there is this idea of AGI that’s smarter than people, which implies that intelligence is some kind of scalar concept where you can have more or less of it, and it’s really significant if you have more or less of it. Do you think that’s basically a correct way to think about intelligence, or should we just not go with that?\n", + "\n", + "**Richard Ngo:**\n", + "Yeah, I think that seems like a good first approximation. There’s this great parody paper, I think it’s called On the Impossibility of Supersized Machines, which says size is not a clear concept. There are different dimensions of size, like height and weight and width, and so on. So, there’s no single criterion of when you count something as bigger than something else. But nevertheless, it does make sense to talk about machines that are bigger than humans. And so in roughly the same way, I think it makes sense to talk about machines that are more intelligent than humans. I’d characterize it as something which draws from a long intellectual history of realizing that humans were like a little bit less special than we used to think, that we’re not in the center of the universe, we’re not that different from other animals, and now that we aren’t at the pinnacle of intelligence.\n", + "\n", + "**Daniel Filan:**\n", + "Okay, why should we think of intelligence as this kind of scalar thing?\n", + "\n", + "**Richard Ngo:**\n", + "One way you can do it is, you can hope that we’ll come up with a more precise theory of intelligence. I like some metaphors from the past. So when we think about the concept of energy, or when we think about the concept of information or even the concept of computing, these were all things that were very important abstractions for making sense of the past. Like you can think about the industrial revolution as this massive increase in our ability to harness energy. Even though, at the time the industrial revolution started, we didn’t really have a precise concept of energy. So, one hope is to say, \"Well, look, we’re going to get a more precise concept as time goes by, just as we did for energy and computation and information. And whatever this precise concept is going to be, we’ll end up thinking that you could have more or less of it.\"\n", + "\n", + "**Richard Ngo:**\n", + "Probably the more robust intuition here is just like, it sure seems that if you make things faster, if you make brains bigger, they just get better at doing a bunch of stuff. And it seems like that broad direction is the dimension that I want to talk about.\n", + "\n", + "**Daniel Filan:**\n", + "Okay, so that’s intelligence. I guess, we’re also talking about general intelligence, which I think you contrast with narrow intelligence. Where, if I recall correctly, a narrow intelligence is something that can do one specific task, but a general intelligence is something that can do a wide variety of tasks. Is that approximately right?\n", + "\n", + "**Richard Ngo:**\n", + "Yeah, that seems right. I think there are a couple of different angles on general intelligence. So, one angle that is often taken is just thinking about being able to deal with a wide range of environments. And that’s an easy thing to focus on, because you can very clearly imagine having a wide range of different tasks. Another take on generality you might have is that generality requires the ability to be very efficient in learning. So being able to pick up a new task with only a couple of different demonstrations. And another one you might have is that generality requires the ability to act coherently over long periods of time. So you can carry out tasks on timeframes not just of hours, but on days, or weeks, or months, or so on.\n", + "\n", + "**Richard Ngo:**\n", + "I think these all kind of tie together, because fundamentally, when you’re doing something in a narrow domain, you can memorize a bunch of heuristics. Or you can encode all the information within the system that you’re using, like the weights of a neural network. Whereas, when you’ve got either long time horizons, or a wide range of environments, or not very much data, what that means is you need something which isn’t just memorized, which is in some sense able to get more out of less.\n", + "\n", + "### AGI: what and how \n", + "\n", + "**Daniel Filan:**\n", + "Okay, cool. And I guess we’re going to take it as given for this discussion that we should expect to get AGI at some point. For people who are wondering, who are perhaps doubtful about that, can you give a sense of why you think that might be possible?\n", + "\n", + "**Richard Ngo:**\n", + "Yeah. So I think there are a couple of core intuitions which seem valuable. So the first one is that there are a lot of disanalogies between neural networks and human brains, but there are also a lot of analogies. And so to me it seems overconfident to think that scaling up neural networks—with presumably a bunch of algorithmic improvements, and architectural improvements, and so on—but once you’re using as much compute, either as an individual human brain, or maybe as all human brains combined, or maybe as the entire history of humanity, which are all milestones that are feasible over a period of decades, at that point it seems like you really need some specific reason to think that neural networks are going to be much worse, or much less efficient than human brains in order to not place a significant credence on those systems having a capability of generality in the same way as humans do.\n", + "\n", + "**Richard Ngo:**\n", + "Another intuition here is that a lot of people think about machine learning as a series of paradigm shifts. So we started off with symbolic AI, and then we moved into something more like statistical style machine learning, and now we’re in the era of deep learning. I think this is a little misleading. The way I prefer to think of it is that neural networks were just plugging along, doing their thing the whole time, like the perceptron and the first models of neurons were around before people think of the field of AI as having officially started. And so if you just look at this trend of neural networks getting more and more powerful as you throw more and more compute at them, and as you make relatively few big algorithmic changes, then it starts to seem more plausible that actually you don’t need a big paradigm shift in order to reach AGI. You can just keep scaling things up because that actually has worked for the last 70 years or so.\n", + "\n", + "**Daniel Filan:**\n", + "Okay. So it seems like your case is really based on this idea that we’re going to have modern machine learning or deep learning, and we’re going to scale it up. We’re going to scale it up in some ways, maybe we’re going to make discrete changes in algorithms, and architectures, and other ways. In this process of scaling, what similarities do you see as going to be retained from now until AGI? And what important differences do you think there are going to be?\n", + "\n", + "**Richard Ngo:**\n", + "Yeah. So I think the core similarity: Neural networks seem like they’re here to stay. Reinforcement learning seems like it’s here to stay. Self-supervised learning seems pretty fundamental as well. So these all feel very solid. Then I think the fundamental question is something like, how much structure do you need to build into your algorithms versus how much structure can you meta-learn, or learn via architectural search, or things like that. And that feels like a very open question to me. It seems like in some cases we have these very useful algorithms, like Monte Carlo tree search [MCTS]. And if you can build that into your system, then why wouldn’t you do that? And it seems really hard to learn something like MCTS.\n", + "\n", + "**Daniel Filan:**\n", + "And MCTS is just sort of randomly looking at, okay, if I take various sequences of actions, how good is that likely to be? Maybe I randomly pick actions, or pick actions that I think are more likely to be optimal, or that kind of thing.\n", + "\n", + "**Richard Ngo:**\n", + "Right, exactly. But you’ve got these different impulses. On one hand, we’ve got a few algorithms, or a few ways of designing networks, that seem very clever, and maybe we can come up with more clever things. On the other hand, the idea of The Bitter Lesson from Rich Sutton, which is that most clever innovations don’t really last. And yeah, so I feel very uncertain about how much algorithmic progress you need in order to continue scaling up AI.\n", + "\n", + "**Daniel Filan:**\n", + "Okay. So a question I have along this front: current AI systems at least seem like a lot of what they’re doing is recognizing patterns in a sort of shallow way. And maybe when you have a neural network and you want to get it into a new domain, you need to expose it to the new patterns until it can pick them up. And maybe it draws something on what the old patterns were like. But it seems, at least to a lot of people, to be very pattern-based. And some people have this intuition that something more like reasoning is going to be needed, where within the neural network it’s understanding its environment and doing things like MCTS play-outs inside its head. And that once you have this kind of thing, you won’t need to be exposed to that many patterns until you understand the nature of what’s going on.\n", + "\n", + "**Daniel Filan:**\n", + "So I’m wondering to what degree does this difference between pattern recognition versus real reasoning play in your thinking? And do you think that real reasoning is necessary and/​or do you think that we’ll get it in things roughly like current neural networks?\n", + "\n", + "**Richard Ngo:**\n", + "I think that, yeah, what you call real reasoning is definitely necessary for AGI. Having said that, I think pattern recognition is maybe more fundamental than a lot of people give it credit for where, when humans are thinking about very high level concepts, we do just use a bunch of pattern recognition. Like when you think about, I guess, great scientists thinking in terms of metaphors or intuitions from a range of different fields, Einstein imagining thought experiments and so on, mathematicians visualizing numbers in terms of objects and the interactions between those objects, these all feel like types of pattern recognition. I’m gesturing towards the idea of this book by Lakoff called Metaphors We Live By, which is basically arguing that a huge amount of human cognition happens via these types of drawing connections between different domains. Now, of course, you need some type of explicit reasoning on top of that, but it doesn’t seem like the type of thing which necessarily requires new architectures, or us to build it in explicitly.\n", + "\n", + "**Richard Ngo:**\n", + "There’s another book which is great on this topic, called The Enigma of Reason, which kind of fits these ideas together: explicit reasoning and pattern matching. It basically argues that explicit reasoning is just a certain type of pattern matching. It’s the ability to match patterns that have the right type of justification. I’m not doing it credit here, so you should check that out if you’re interested. But basically, I think that these two things are pretty closely related, and there’s not some fundamental difference.\n", + "\n", + "### Single vs collective AI minds \n", + "\n", + "**Daniel Filan:**\n", + "Okay. So moving on a bit, in the report, you talk about different ways AGIs could be. One of them is, you have this single neural network maybe that’s an AGI, which I think a lot of people are more familiar with. And then you also talk about collective AGIs. Maybe it’s an AGI civilization where the civilization is intelligent, or something like that. Could you describe those, and tell us how important are those to think about, if people don’t normally think about them?\n", + "\n", + "**Richard Ngo:**\n", + "I guess the core intuition behind thinking about a single AGI is that our training procedures work really well when you train end-to-end. You take a system, you give it a feedback signal, and you just adjust the whole system in response to that feedback signal. And what you get out of it is basically one system where all the parts have been shaped to work together by gradient descent. So that’s a reason for thinking that, in terms of efficiency, having one system is going to be much more efficient than having many different systems trying to work together.\n", + "\n", + "**Richard Ngo:**\n", + "I think the core intuition for thinking about a collection of AGIs is that it’s very cheap to copy a model after you’ve learned it. And so if you’re trying to think about the effects that building a single AGI will have, it seems like the obvious next step is to say, \"Well, you’re going to end up with a bunch of copies of that, because it’s very cheap to make them. And then now you’re going to have to reason about the interactions between all of those different systems.\" I think that it probably doesn’t make a huge difference in terms of thinking about the alignment problem, because I don’t think you’re going to get that much safety from having a large collection of AGIs compared to just having one AGI. It feels like if the first one is aligned, then the collection of many copies of it is going to be as well. And if it’s not, then they won’t. But it does seem pretty relevant to thinking about the dynamics of how AGI deployment might go in the world.\n", + "\n", + "**Daniel Filan:**\n", + "Okay, cool. And I guess one question I have related to this, that sort of points to this idea that instead of thinking just about each AGI, or each neural network or whatever, being intelligent, we should also think of the intelligence of the whole group. And so I have this colleague, Dylan Hadfield-Menell, who’s very interested in this idea that when we’re thinking of training AIs to do things that people want, we should think about having a human AI system be rational in the sense of conserving resources: that whole system should achieve the human goal. So I’m wondering like what thoughts do you have about the locus of intelligence: should we think of that as being single AI systems, multitudes of AI systems, or some combination of humans and AIs?\n", + "\n", + "**Richard Ngo:**\n", + "I think that the best outcome is to have the locus of intelligence effectively being some combination of humans and AIs. To me that feels like more of a governance decision rather than an alignment decision, in the sense that I expect that increasingly as we build the AGIs, they’re going to take on more and more of the cognitive load of whatever task we want them to do. And so if there’s some point we were going to stop and say like, \"No, we’re going to try and prevent the AGIs from assuming too much control\" then, I think, fundamentally, that’s got to be some sort of regulatory or governmental choice. I don’t think it’s the type of thing that you’d be able to build in on a technical level, is my guess.\n", + "\n", + "**Richard Ngo:**\n", + "And then when it comes to many AGIs, I think people do underrate the importance of, in particular, cultural evolution when it comes to humans. Where that’s one way in which the locus of intelligence is not in a single person, it’s kind of spread around many people. Yeah, so that’s some reason to suggest that the emergent dynamics of having many AGIs will play an important role. But I think most of the problem comes in talking about a single system. And like, if you can have guarantees about what a single system will do, then you can work from there to talk about the multi-agent system. Basically, I think the claim is that having multi-agent system makes things harder, but some people have argued that it kind of makes things easier, that it means we don’t need to worry about the alignment problem for a single agent case. And I think that’s incorrect. I think, start off by solving it for the single agent case, without relying on any messy multi-agent dynamics to make things go well.\n", + "\n", + "## AGI in practice \n", + "\n", + "### Impact \n", + "\n", + "**Daniel Filan:**\n", + "So we sort of hinted at this idea of an alignment problem, or various risks. Some people seem to think that if we had AGI, this would be a really big deal. Do you think that’s right? And if so, why?\n", + "\n", + "**Richard Ngo:**\n", + "Yeah, I think that’s right. I think there’s the straightforward argument which is, humans can do incredible things because of our intelligence. If you have AGIs, then they’ll be able to do incredible things either much faster, or in a way that builds on themselves, and this will lead to types of scientific and technological advancement that we can’t really even imagine right now. Maybe, as one thought experiment I particularly like, imagine taking 10,000 scientists, like 10,000 of the best scientists and technologists, and just accelerating them by a factor of 1,000, just speeding up the way they can think, speeding up the types of progress that they make, and speeding up, I guess, their computers that they’re working on as well. And what could they do after the equivalent of 500 or 1,000 years?\n", + "\n", + "**Richard Ngo:**\n", + "Well, they would have some disadvantages, right? They wouldn’t be able to engage with the external world nearly as easily. But I think if you just look back 500 years ago, and you think how far have we come in 500 years through this process of scientific and technological advancement, it feels like it’s kind of wild to think about anywhere near the same amount of progress happening over a period of years or decades. So, that’s one intuition that I find particularly compelling.\n", + "\n", + "### Timing \n", + "\n", + "**Daniel Filan:**\n", + "Okay. Seems like a pretty good deal. When do you think we will get AGI, and why do you think it’ll take that long?\n", + "\n", + "**Richard Ngo:**\n", + "Yeah, so I don’t have particularly strong views on this. So to a significant extent, I defer to the report put out by Ajeya Cotra at Open Philanthropy which, if I recall correctly, has a median time of around 2045 or 2050. I think that report is a pretty solid baseline. I don’t think we should have particularly narrow credences around that point in time. I guess, it feels like there are some high level arguments that sway me, but they ultimately kind of cancel out. So one high level argument is just, look, there are a lot of cases in the past where people were wildly overconfident about the technology needing to take a long time. So, the classic example being the predictions by the Wright brothers about how long they’d take to build an airplane, where they thought it was 50 years away, I think two years before they built it. People thought it would take way longer than it did to land people on the moon. I think Lord Rutherford thought that the nuclear reaction was kind of moonshine and nonsense, and that was only a few years before, or maybe even after, they’d made the most critical breakthroughs. So, all of these, they feel like strong intuitions that kind of weigh in the direction of not ruling out significantly earlier timelines. On the other hand, these do feel a little bit cherry-picked and it feels like there are many other examples where people just like see one plausible path to getting to a point and don’t see the obstacles that are going to slow them down a lot, because the obstacles are not as salient.\n", + "\n", + "**Richard Ngo:**\n", + "So probably people 50 years ago were wildly overconfident about how far rocket technology and nuclear power would get. And these are pretty big deals. It would be very significant if we could do asteroid mining right now, or if we had very cheap nuclear power, but there were a whole bunch of unexpected obstacles even after the initial early breakthroughs. So, I think these situations weigh against each other, and the main effect is just to make me pretty uncertain.\n", + "\n", + "**Daniel Filan:**\n", + "All right. Cool. And a related question that people have: some people think that we might need to make preparations before we get AGI. And if AGI will come very suddenly, then we should probably do those preparations now. But if it’ll be very gradual and we’ll get a lot of warning, maybe we can wait until AGI is going to come soon-ish to prepare to deal with it.\n", + "\n", + "**Daniel Filan:**\n", + "So on the question of how sudden AGI might be, do you think that it’s likely to happen all at once without much warning, or do you think that there’s going to be a very obvious gradual ramp up?\n", + "\n", + "**Richard Ngo:**\n", + "I think there’s going to be a noticeable and significant ramp up. Whether that ramp up takes more than a couple of years is hard to say. So, I guess the most relevant question here is something like, how fast is the ramp up compared with the ability of, say, the field of machine learning to build consensus and change direction, versus how fast does it compare with the ability of world governments to notice it and make appropriate actions, and things like that? While I’m pretty uncertain about how long the ramp up might take, it feels pretty plausible that it’s shorter than the other comparable ramp ups that I just mentioned. So that feels like the decision relevant thing. If the world could pivot in a day or two to really taking a big threat seriously, or really taking technological breakthrough seriously, then that would be a different matter.\n", + "\n", + "**Richard Ngo:**\n", + "But as it is, how long does it take for the world to pivot towards taking global warming seriously? A matter of decades. Even coronavirus, it took months or years to really shift people’s thinking about that. So, that’s kind of the way I think about it. There’s no real expectation that people will react in anywhere near a rapid enough way to deal with this, unless we start preparing in advance.\n", + "\n", + "### Creation \n", + "\n", + "**Daniel Filan:**\n", + "Cool. Another question I have just in terms of what it might look like to get AGI. There are two scenarios that people talk about. One scenario, sort of related to the idea of collective AGIs and stuff, there’s one strain of thought which says probably what will happen is around the time that we get AGI, a bunch of research groups will figure it out at approximately the same time interval, and we should imagine a world where there are tons of AGI systems. Maybe they’re competing with each other, maybe not, but there are a bunch of them and we have to think about a bunch of them. On the other hand, some people think, probably AGI is one of these things where either you have it or you don’t to a pretty big extent, and therefore there’s going to be a significant period of time where there’s just going to be one AGI that’s way more relevant than any other intelligent system. Which of these scenarios do you think is more likely?\n", + "\n", + "**Richard Ngo:**\n", + "I don’t think it’s particularly discrete, so I don’t think you’ve either got it or you don’t. I think that the scenario that feels most likely to me (although, again, with pretty wide error bars) is that you have the gradual deployment of systems that are increasingly competent on some of the axes that I mentioned before. So, in particular, being able to act competently over longer time horizons, and being able to act competently in new domains with fewer and fewer samples, or less and less data. So, I expect to see systems rolled out that can do a wide range of tasks better than humans over a period of five minutes or a period of an hour, or possibly even over a period of a day, before you have systems that can do the strategically vital tasks that take six months or a year: things like long term research, or starting new companies, or making other large scale strategic decisions.\n", + "\n", + "**Richard Ngo:**\n", + "Then I expect there to be a push towards increasingly autonomous, increasingly efficient systems. But then once we’ve got those systems starting to roll out, I guess I think of it in terms of orders of magnitude. So, the difference between being able to act competently over a day versus being able to act competently over a week versus six months or so, is we should expect that going from a week to a couple of months is not much harder than going from a day to a week. Each time you jump up in order of magnitude, it feels like roughly speaking, we should expect it to be a similar level of difficulty.\n", + "\n", + "**Daniel Filan:**\n", + "Okay. So, does that look like a world where this whole field, one person out of the field managed to figure out how to get their AI to plan on the scale of a year, and the rest of them figured out how to have their AIs plan on the scale of a month or two? And maybe the one that can plan on the scale of a year is just way more relevant to the world than the rest of them?\n", + "\n", + "**Richard Ngo:**\n", + "Yeah, that seems right.\n", + "\n", + "### Risks and benefits \n", + "\n", + "**Daniel Filan:**\n", + "Okay, cool. We’ve mentioned this idea that maybe… well, your report is titled \"AGI Safety from First Principles\", and we’ve talked a bit about the things like the problem of alignment. How good or bad should we expect AGI to be in terms of its impact on the world and what we care about?\n", + "\n", + "**Richard Ngo:**\n", + "It feels like the overall effect is going to be dominated by the possibility of extreme risk scenarios, to me. It feels like it’s hard to know what’s going on in expectation, because you have these very positive outcomes where we’ve solved scarcity and understand how to build societies in much better ways than we currently do, and can make a brilliant future for humanity, and then also the ones where we’ve ended up with catastrophic outcomes because we’ve built misaligned AGIs. So, yeah, hard to say on balance what it is, but it feels like a point in time where you can have a big influence by trying to swing it in the positive direction.\n", + "\n", + "**Daniel Filan:**\n", + "It seems like you think there’s a good chance that we’re going to build AGI that might do something really bad. Does it kill everyone, does it enslave everyone, does it make everyone’s lives 10% worse, or what?\n", + "\n", + "**Richard Ngo:**\n", + "Yeah. I think probably the way I would phrase it is that the AGI gains power over everyone, and power in the sense of being able to decide how the world goes. It feels pretty hard for me to say exactly what happens after that. But it feels like by the time we’ve reached that point, we’ve screwed up. When I say having power, what I mean is having power in a way that doesn’t defer that power back to humans.\n", + "\n", + "**Daniel Filan:**\n", + "Okay. How afraid of that should we be? Because I guess there’s some world where what that looks like is, maybe I should imagine that we have something like the normal economy, except everyone pays 50% of their income to the AGI overlords, and the AGI gets to colonize a whole bunch of stuff, and it buys a whole bunch of our labor. Should we be super afraid of that?\n", + "\n", + "**Richard Ngo:**\n", + "Yeah, I think we should be pretty afraid, in a pretty comparable way to how if you described to gorillas, humans taking over the world, they should be pretty afraid. Now, it’s true that there’s some chance that the gorillas get a happy outcome, like maybe we’re particularly altruistic and we are kind to the gorillas. I don’t think there was any strong reason in advance to expect that humans would be kind to gorillas, and, in fact, there have been many cases throughout history of humans driving other species to extinction, just because we had power over them and we could. So, broadly speaking, if there’s a set of systems that choose to gain power over humans, even just from that fact, we can probably infer that we should be pretty scared.\n", + "\n", + "**Daniel Filan:**\n", + "It seems like AGI is probably going to be a thing that people make, or at least people will make the things that make it. There’s some sort of argument which says, look, probably we won’t build really terrible AGI systems that take over the world, because if anyone foresees that, they won’t do it because they don’t want the world to be taken over. I’m wondering what you think of this argument for \"things will be fine because we just won’t do it\".\n", + "\n", + "**Richard Ngo:**\n", + "I think in some sense, the core problem is that we don’t understand what we’re doing when we build these systems to anywhere near the same level as we understand what we’re doing when we build an airplane, or when we build a rocket, or something like that. So, yeah, in some sense, if you step out and you’re like, \"Why is AI different from other technologies?\" One answer is just that we don’t have a scientific, principled understanding of how to build AI. Whereas for a lot of these other technologies, we do. Now, we could say, \"Well, let’s experiment until we get that.\" Then I think the answer to that would just be, if you really expect AGI to be such a powerful technology, then you might not have that long to experiment. Maybe you have months or years, but that’s not really very long in the grand scheme of things, when you’re trying to figure out how to make a system safe. In particular, it’s not long to make a very new, very powerful type of technology safe. If we really knew what we were doing, in terms of us designing the systems, then I’d feel much better. But as it is, it’s more like our optimizers or our training algorithms are the ones designing the systems, and we’re just kind of setting it up and then letting it run a lot of the time.\n", + "\n", + "**Daniel Filan:**\n", + "It seems like in this story, there’s still some point where somebody says, \"Look, I’m going to set up this system, and there’s a really good chance it’s going to turn into an AGI, and I don’t really understand AGIs, and there are these good arguments that I’ve heard on this podcast that I listen to for how they’re going to be powerful and they’re going to be dangerous, but I’m going to press the button anyway.\" Why do they press the button? Isn’t that irrational?\n", + "\n", + "**Richard Ngo:**\n", + "One answer is just that they don’t really believe the arguments, right? I think it’s easy to not believe arguments when the thing you’re postulating is this qualitatively different behavior from the other systems that have come before. Jacob Steinhardt has an excellent series of blog posts recently, talking about emergent changes in behavior as you scale up AI systems. I think it’s kind of like, it does seem a little bit crazy that as you make the systems more and more powerful, you’re not really changing the fundamental algorithms or so on. But you do get these fundamental shifts in what they’re able to do. So, not just performing well in a narrow domain, but performing well in a wide range of domains. Maybe the best example so far is GPT-3 being able to do few shot learning just by being given prompts.\n", + "\n", + "**Richard Ngo:**\n", + "And that’s much smaller than the types of change you expect as you scale systems up from current AI to AGIs. So, yeah, plausibly people just don’t intuitively believe the scale of the change that they ought to expect to see. Then the second argument is maybe they do, but there are a bunch of competitive motivations that they have. Maybe they’re worried about economic competition, or maybe they’re worried about geopolitical competition. It seems pretty hard to talk about these very large scale things decades in advance, but that’s the sort of fundamental shape of an argument that I think is pretty compelling. If people are worried about getting to a powerful technology first, then they’re going to cut corners.\n", + "\n", + "## Making AGI safe \n", + "\n", + "### Robustness of the agency abstraction \n", + "\n", + "**Daniel Filan:**\n", + "Now I’d like to talk a bit about the technical conversation about making these AGI systems safe. You recently had this conversation with Eliezer Yudkowsky, an early proponent of the idea there that there might be existential risk from AGI. One thing that came up is this idea of goal-directed agents, where I think Eliezer had this point of view that was very focused on this idea that we’re going to get these AI systems, and they will sort of coherently shape the world in a certain way. He thinks that there’s just one natural way to coherently shape the world for simple goals. Where it seems like in your point of view, you think about these abilities of agents, like self-awareness, ability and tendency to plan, picking actions based on their consequences, how sensitive they are to scale, how coherent they are, and how flexible they are. In your view, it seems like you think about agents as sliders on these fronts. Maybe we’ll have a low value on one of the sliders. I’m wondering what your thoughts are about the relationship between these ways of thinking, and where you would rely on one versus the other.\n", + "\n", + "**Richard Ngo:**\n", + "I think the first thing to flag here is that Eliezer and I agree to a much greater extent than you might expect from just reading the debate that we had. The core idea that he has, that there’s something fundamentally dangerous about intelligence, is something that I buy into. I think that the particular things that I want to flag on that front are this idea of instrumental reasoning. Suppose you’re an obedient AI, and a human asks you to achieve a goal. You have to reason about what intermediate steps you’re going to take, you need to be able to plan out sub goals, and then achieve those sub goals, and then go on and achieve this far goal.\n", + "\n", + "**Richard Ngo:**\n", + "It feels like that’s very core to the concept of general intelligence. In some sense, you’ve just got this core ability, and we’re like, \"But also please don’t apply it to humans. Please don’t reason about humans instrumentally. Please don’t make sub goals that involve persuading me or manipulating me.\" I think Eliezer is totally right that as you scale this ability up, then it becomes increasingly unnatural to have this sort of exception for humans in the instrumental reasoning that the system is doing. You’ve got this pressure towards doing instrumental reasoning towards achieving outcomes, and in some sense, what alignment is trying to do is carve out this special zone for human values, which says, \"No, no, no, don’t do the intelligence thing at us in that way.\"\n", + "\n", + "**Richard Ngo:**\n", + "I’d also flag in another core idea here is the idea of abstraction, the idea that intelligence is very closely linked to being able to zoom out and see a higher level pattern. Again, that’s something that when we want to build bounded agents, we want to try and avoid. We want to say, \"We’ve given you a small scale goal,\" or, \"We’ve trained you to achieve a small scale goal. Please do not abstract from that into wanting to achieve a much larger scale goal.\" Again, we’re trying to carve out an exception, because most of the time we want our agents to do a lot of abstraction. We want them to be thinking in very creative, flexible ways about how the world works. It’s just that in the particular case of the goals we give them, we don’t want them to generalize in that way.\n", + "\n", + "**Richard Ngo:**\n", + "So, that’s my summary of some of these ideas that I’ve gotten from Eliezer, at least, where he has this concept that in the limit, once you push towards the heights of greater general intelligence, it becomes incredibly hard to prevent these capabilities from being directed at us in the ways that we don’t want. It feels like maybe the core disagreement I have is something like, how tight is this abstraction? In the sense of, how much can we trust that these things are correlated with each other, not just in the regime of highly superintelligent systems, but also in the regime of systems that are slightly better than humans, or even noticeably significantly better than humans at doing a wide range of tasks, like intellectual research, like jobs in the world, like doing alignment research specifically, things like that.\n", + "\n", + "**Richard Ngo:**\n", + "I guess my position is just either I don’t understand why he thinks that his claims about the limit of intelligence are also going to continue to apply for the relevant period as we move towards that point, or else maybe he’s trusting too much in the abstraction and failing to see ways in which reality might be messier than he thinks is going to be the case.\n", + "\n", + "**Daniel Filan:**\n", + "Okay. So, am I right to summarize that as you thinking that, indeed, there’s some sort of limit where in order to be generally competent, if you have enough of the things that we call intelligence, then it’s very hard for you to not think about humans instrumentally and abstract your goals to larger scales and whatever, but maybe near a human level, we can still do it and it’ll be fine?\n", + "\n", + "**Richard Ngo:**\n", + "That’s right. We might hope that in particular, one argument that feels pretty crucial to me is the idea that our comparative advantage, as humans, is towards a whole bunch of things that are not particularly, let’s say, aligned from the perspective of other species. So, we have a strong advantage at doing things like expanding to new areas, like gathering resources, like hunting and fighting and so on. We’re not very specialized at things like doing mathematical research, reasoning about alignment, reasoning about economics, for example, in ways that make our societies better. That’s just because of the environment in which we evolved.\n", + "\n", + "**Richard Ngo:**\n", + "So, it seems very plausible to me that as we train AIs to become increasingly generally intelligent, eventually, they’re going to surpass humans at all of these things, but the hope would be that they surpass humans at the types of things that are most useful and least worrying, before they surpass humans in terms of the power seeking behavior that we’re most worried about. So, I guess, yeah, a question of differing comparative advantages, even though eventually once they get sufficiently intelligent, they’ll outstrip us at all of these things.\n", + "\n", + "### Pivotal acts \n", + "\n", + "**Daniel Filan:**\n", + "Yeah. One concept that I think is lying in the background here is this idea of a pivotal act. A listener might listen to that and think, \"Well, it sounds like you’re saying that for a while, we’ll have slightly superintelligent AI, and that’ll be fine, but then when we get the really superintelligent AI, that will kill us. Why should I feel comforted by that?\"\n", + "\n", + "**Richard Ngo:**\n", + "Yeah.\n", + "\n", + "**Daniel Filan:**\n", + "So, some people have this idea that, look, when we get really smart AI, step one is to use it to do something that means that we don’t have to worry about risk from artificial general intelligence anymore. People tend to describe drastic things. To give listeners a sense, I think in this conversation with Yudkowsky, Yudkowsky gave the example of melting every GPU on earth. I’m wondering, how much do you buy the idea of a pivotal act being necessary, and do you think that being intelligent enough in the way that you can do some kind of pivotal act is compatible with the kinds of intelligence where you’re coherent enough to achieve things of value, but you forgot to treat humans as instrumental things to be manipulated in service of your goals?\n", + "\n", + "**Richard Ngo:**\n", + "I don’t think we have particularly strong candidates right now for ways in which you can use an AGI to prevent scaling up to dangerous regimes. But I think there are plausible things that seem worth exploring, that are maybe a little bit less dramatic sounding than Eliezer’s example. So, in the realm of alignment research, you might have a system that can make technical progress on mathematical questions of the sort that are related to AI alignment research. So, one option is automating the theoretical alignment research. Another option which is associated with proposals like amplification, reward modeling, and debate is just to use these systems to automate the empirical, practical side of alignment research by giving better feedback.\n", + "\n", + "**Richard Ngo:**\n", + "Then on the governance side, I think just having a bunch of highly capable AIs in the world is going to prompt governments to take risks from AGI a lot more seriously. I don’t think that the types of action that would be needed to slow down scaling towards dangerous regimes are actually that discontinuous from the types of things we see in the world today. So, for example, global monitoring of uranium and uranium enrichment to prevent proliferation of nuclear weapons. I think, indeed, there’s a lot of cultural pressure against things like building nuclear power, and a wide range of other technological progress that’s seen as dangerous, so I feel uncertain about how difficult or extreme governance interventions would need to be in order to actually get the world to think, \"Hey, let’s slow down a bit. Let’s be much more careful.\" But to me, it still feels plausible that pivotal action is a little bit of a misnomer. It’s more just like the world handling the problem as the world wakes up to the scale and scope of the problem.\n", + "\n", + "**Daniel Filan:**\n", + "So, it seems like in your way of thinking, the thing that stops the slide to incredibly powerful, unfriendly AGI, is we get AGIs to help do our AI safety research for us. Is that about right? As well as a bunch of governance actions?\n", + "\n", + "**Richard Ngo:**\n", + "Right. Yeah. That feels like the default proposal that I’m excited about. But in terms of the specifics of the proposals, I can’t point to any particular thing and say, \"This one is one that I think would work right now.\"\n", + "\n", + "**Daniel Filan:**\n", + "Sure. I guess if I imagine effective safety research, it seems to involve both pretty good means-ends reasoning, like, you have to reason about these systems that we’re going to deploy and what they’re going to do, and maybe they’re going to get roped into helping with the safety research. We’ll have to think about how they’ll research or have some invariants about that are going to be maintained or something. So you have to have pretty good means-ends reasoning. And you also have to be thinking about humanity, in order to know what counts as safety research, versus research to ensure that the AGI is blue, or some other random property that nobody cares about. And I think there’s some worry that, look, as long as you have that amount of goal orientation in order to get you doing really good research, and that amount of awareness of humans, there’s some worry that that combination of attributes is itself enough to think about humans instrumentally in the dangerous way. I’m wondering what you think about that.\n", + "\n", + "**Richard Ngo:**\n", + "I agree that’s an argument as to why we can’t just take a system, say, \"go off, please solve the alignment problem\" and then have it just come back to us and give us a solution. So I think yeah, in some sense, many of the alignment proposals that are on the table today are ways of trying to mitigate the things you’re discussing. It’s hard to say like how much of a disagreement there is here, because you know, I do think, you know all of the things you said are just reasons to be worried, but then it feels like…\n", + "\n", + "**Richard Ngo:**\n", + "I think this partly ties back to the thing I was saying before about like the core problem being that we just don’t understand the way that these systems are developed. So it’s less like you have to do a highly specific thing with your system in order to make it go well, and more like you just need to either you need to have a deeper understanding, which is kind of a bit more like intellectual work than going out and doing stuff in the world, or maybe you need to just need to scale up certain kinds of supervision. Which again, I don’t currently see the reasons why this is necessarily infeasible. It feels like there’s a lot of scope here for promising proposals.\n", + "\n", + "## AGI safety concepts \n", + "\n", + "### Alignment \n", + "\n", + "**Daniel Filan:**\n", + "Let’s move on to this thing that we’ve mentioned a couple of times now called alignment. First of all, what do you mean by \"alignment\" or, \"misaligned AI systems\"?\n", + "\n", + "**Richard Ngo:**\n", + "To a first approximation, when I say \"aligned\" I mean \"obedient\". So, a system that when you give it an instruction, it’ll follow the intention of that instruction and then come back and wait for more instructions. That’s roughly what I mean. And by \"misaligned\", roughly what I mean is power seeking. A system that is trying to gain power over the world for itself. Either in order to achieve some other goal that humanity wouldn’t want, or just for the sake of it: in the same way that some humans are sort of intrinsically power seeking, you might have an AI that’s intrinsically power seeking. Those are the two more specific concepts I usually think about.\n", + "\n", + "**Daniel Filan:**\n", + "How important is it for safe AGI to solve the technical problem of making AIs that are aligned? Do you think that that has like a massive role in making our AGI’s safe, or a minor role? Do you think it’s basically the whole problem, or 10% of the problem, or something?\n", + "\n", + "**Richard Ngo:**\n", + "Is the question something like what proportion of the difficulty is alignment versus governance work?\n", + "\n", + "**Daniel Filan:**\n", + "I think people, there are some people you think that, oh, you’ll need to do alignment, and also some other stuff. I don’t know if it’s governance. I don’t know if it’s, governance of thinking about how the AIs are going interact, and maybe some people really want them to coordinate? There might be a variety of things there.\n", + "\n", + "**Richard Ngo:**\n", + "Yeah. I guess I feel like it’s most of the problem, but nowhere near all of it. If I think about the ways in which I’m concerned about existential risk from advanced AI, probably the split is something like 50% worried about alignment, 25% worried about governance failures related to alignment, and then 25% worried about straightforward misuse, or conflicts sparked by major technological change.\n", + "\n", + "**Daniel Filan:**\n", + "So a concern that I think some people have with the idea of AI alignment is this fear that we’re going to create this ability for these really powerful people to create AI systems that just do what they want, that are super obedient to them. And I think some people have this idea that, oh, we’ve just created this effective machine to turn people into tyrants, or totalitarian overlords. How worried are you about this?\n", + "\n", + "**Richard Ngo:**\n", + "I think that yeah, it seems pretty worrying that advances in technology lead to dramatic increases in inequality, and a big part of AI governance should be setting up structures and systems to ensure that these systems, if they’re aligned, are then used well. I think there’s work on this. Cullen O’Keefe, for example, has a paper on windfall clauses, which talks about the ways in which you might try and redistribute benefits from AGI. More broadly, I think there’s a bunch of various unsolved questions to do with not just corporate governance, but also global governance. Ultimately, it’s not clear to me what the alternative to addressing these questions is. I think that it would be nice if we could kind of like mitigate all of these problems at once, but it feels like we’re just going to have to do the hard work of, setting up as many structures and safeguards as we can.\n", + "\n", + "**Daniel Filan:**\n", + "So, I guess some people might think, look, the alternative is try and think hard until you come up with a plan other than \"create an AI that’s really aligned to an individual\", or maybe come up with a technical plan to create an AI that’s aligned with humanity, or with objective moral truth, or something.\n", + "\n", + "**Richard Ngo:**\n", + "I guess I feel pretty pessimistic about not just having to solve alignment, but also having to solve morality. For example, when I say obedient, it doesn’t need to be the case that the system is obedient to any given individual, right? Probably you want to train it so that it’s obedient to, ideally speaking, individuals who have the right certifications. Such as being democratically elected, or things like that. And along with that, hopefully it’s obedient only within certain bounds of what actions they’re meant to take. So like, in an ideal world, if I could have all the alignment desiderata I wanted, I’d set that up in a much better way, but I do think that this core idea of obedience, it feels valuable to me because, yeah, the problem of politics is hard. The problem of ethics is hard. If you can solve the problem of making a system obedient, then we can try and leverage all the existing solutions we have to governance. Things like all the systems and structures that have been built up over time to try and figure out how we’re going to deploy advanced technologies. I don’t want people to try and have to reason these things through from first principles, while they’re trying to build aligned AGIs.\n", + "\n", + "### Transparency \n", + "\n", + "**Daniel Filan:**\n", + "Another thing that you mentioned in AGI Safety from First Principles is transparency, as this important part of AGI alignment. So could you first say, perhaps briefly, what role you see transparency research as playing?\n", + "\n", + "**Richard Ngo:**\n", + "I’ll speak differently for the wider field, and then for myself. So I think in the wider field, it’s playing a pretty crucial role right now. In terms of transparency is in some sense, one of the core underlying drivers of many proposals for alignment. I’d say the other core driver here is just using more human data, just like trying to get more feedback from humans so that we can use that to nudge systems towards fulfilling human preferences better. So yeah, a lot of research agendas are kind of assuming a certain amount of interpretability or transparency. I’m using those interchangeably. For my own part, I defer to other people to some extent on how much progress we’re going to make, because it does seem like there’s been pretty impressive progress so far. I feel a little confused about how work on interpretability could possibly scale up as fast as we’re scaling up the most sophisticated models.\n", + "\n", + "**Richard Ngo:**\n", + "When I think about trying to understand the human brain to a level that’s required to figure out if a thought that a human is having is good or bad, that seems very hard. People have been working at it for a very long time. And now of course there are a bunch of big advantages that we have when we do interpretability research on neural networks. Like we have full read/​write access to the neurons, but we also have a bunch of disadvantages as well. Like the architecture changes every couple of years. So you have to switch to a new system that might be built quite differently, and you don’t have introspective access, or necessarily very much cooperation from the system as you are trying to figure out what’s going on inside it.\n", + "\n", + "**Daniel Filan:**\n", + "Sure.\n", + "\n", + "**Richard Ngo:**\n", + "So, on balance I feel personally kind of pessimistic, but at the same time it seems like something that we should try and make as much progress on as we can. It feels very robustly good to know what’s going on within our systems.\n", + "\n", + "**Daniel Filan:**\n", + "So, if you’re pessimistic, do you think we can do without it, or do you think we’ll need it, but it’s just very hard and it’s unlikely we’ll get it?\n", + "\n", + "**Richard Ngo:**\n", + "I have a pretty broad range of credence over how hard the alignment problem might be. You know, there’s a reasonable range in which interpretability is just necessary or, something equivalently powerful is necessary. There’s also ranges in which it isn’t. And I think I focus a bit more on the ranges of difficulty in which it’s not necessary, just because those feel like the most tractable places to pay attention. So I don’t really have a strong estimate of the ratio between those, let’s say.\n", + "\n", + "### Cooperation \n", + "\n", + "**Daniel Filan:**\n", + "Okay, sure. Another thing to talk about is this idea of AI cooperation. Vincent Conitzer has recently started a seminar series on this. I believe Andrew Critch wrote a thing basically arguing that even if you solve AI alignment, if you’re going to have a whole bunch of AI systems, then solving the problem of making sure that the interactions between them don’t generate externalities which are dangerous to humans or something, is even harder than solving the alignment problem [Here’s a note from future Daniel: to the best of my knowledge, there’s no written piece where Andrew Critch makes this argument. He has said via personal communication that he prefers to not debate which problem is harder, and that he would like people working on both.]. So I’m wondering, what do you think about the worry about making sure that AIs coordinate well?\n", + "\n", + "**Richard Ngo:**\n", + "I guess it feels like mainly the type of thing that we can outsource to AIs, once they’re sufficiently capable. I don’t see a particularly strong reason to think that systems that are comparably powerful as humans, or more powerful than humans, are going to make obvious mistakes in how they coordinate. You have this framing of AI coordination. We could also just say politics, right? Like we think that geopolitics is going to be hard in a world where AIs exist. And when you have that framing, you’re like, geopolitics is hard, but we’ve made a bunch of progress compared with a few hundred years ago where there were many more wars. It feels pretty plausible that a bunch of trends that have led to less conflict are just going to continue. And so I still haven’t seen arguments that make me feel like this particular problem is incredibly difficult, as opposed to arguments which I have seen for why the alignment problem is plausibly incredibly difficult.\n", + "\n", + "## Optima and selection pressures \n", + "\n", + "**Daniel Filan:**\n", + "All right. I’d like to get back to a thing we talked about a bit earlier, which is this question about how much we should think about the optima of various reward functions, or the limit of intelligence or something. I see you as thinking that people focus perhaps too much on that, and that we should really be thinking about selection pressures during training. I’m wondering what mistakes do you think people make when they’re thinking in this framework of optima?\n", + "\n", + "**Richard Ngo:**\n", + "I think there are kind of two opposing mistakes that different groups of people are making. So Eliezer, and MIRI more generally, it really does feel they’re thinking about systems that are so idealized that insights about them aren’t very applicable to systems like the full first AGIs we’ll build, or ones that are a little bit better than humans at making intellectual progress. There are a bunch of examples of this. I think AIXI is kind of dubious. One of the metaphors I use sometimes is that it’s like trying to design a train by thinking about what happens when a train approaches the speed of light. You know, it’s just not that helpful. So that’s one class of mistakes that I’m worried about, related to the idea that you can just extrapolate this idea of intelligence until it’s to the infinite limit. Like there is such a thing as perfect rationality, for example, or the limit of approaching perfect rationality makes sense.\n", + "\n", + "**Richard Ngo:**\n", + "So that’s one mistake I’m worried about. I think on the other hand, it feels like a bunch of more machine learning focused alignment researchers don’t take the idea of optimization pressure seriously enough. It feels like there are a few different strands of research that just reroute the optimization pressure, or block it from flowing through the most obvious route, but then just leave other ways for it to cause the same problems. Probably the most obvious example of this to me is the concept of myopia, which especially Evan Hubinger is pretty keen on, and a few other researchers are as well. And to me, it seems you can’t have it both ways. If you’ve got a system that is highly competent, then there must have been some sort of pressure towards it achieving things on long timeframes.\n", + "\n", + "**Richard Ngo:**\n", + "So even if it’s like nominally myopic, I haven’t seen any particular insight as to how you can have that pressure applying, without making the agent actually care about or pursue goals over long time horizons. And I think the same thing is true with cooperative inverse reinforcement learning, like where it feels to me like it just kind of reroutes the pressure towards instrumental reasoning in a slightly different way, but it doesn’t actually help in preventing that pressure from pushing towards misaligned goals. The key thing that I would like to see from these types of research agendas, and to some extent also stuff from ARC like imitative generalization and eliciting latent knowledge, is the core reason why this isn’t just reframing the problem. I want to know which part is actually doing the work of preventing the optimization pressure towards bad outcomes. Because I feel pretty uncertain about that for a lot of existing research agendas.\n", + "\n", + "**Daniel Filan:**\n", + "Let’s talk about cooperative inverse reinforcement learning. It’s a subject I’m relatively more familiar with. So I think the idea, as I understand it, is something like look, we’re going to think about human AI interaction like so: the human has a goal, and like the human/​AI system somehow has to optimize for that goal. And so the alignment is sort of coming from it being instrumentally valuable for the AI system to figure out what the human goal is, and then to optimize for it. And so you get alignment because the AI is doing what you want in that way. I’m wondering like, where do you think this is missing the idea of optimization pressure?\n", + "\n", + "**Richard Ngo:**\n", + "So the way that Stuart Russell often explains his ideas is that the key component is making an AI that’s uncertain about human preferences.\n", + "\n", + "**Daniel Filan:**\n", + "Yep.\n", + "\n", + "**Richard Ngo:**\n", + "But the problem here is that the thing that we really want to do is just point the AI at human preferences at all, make it so that it is in fact optimized in the direction of fulfilling human preferences. Whether or not it actually has uncertainty about those things, about what humans care about, that uncertainty is an intermediate problem. Where the fundamental problem is: what signal are we giving it that points it towards human preferences? Or, what are we changing compared with a sort of straightforward setup where we just give it rewards for doing well and penalize it for doing badly? And one thing you might say is look, the thing we’re changing is that we have the model of human rationality.\n", + "\n", + "**Richard Ngo:**\n", + "We have some assumptions about how the human is choosing their actions. And that could be the thing that’s doing the work, but I haven’t seen any model that’s actually making enough progress on this, that it’s plausible that it’s doing the the heavy lifting. Most of the models of human rationality I’ve seen are very simple ones of noisy rationality or things like that. And so if that model isn’t doing the work, then what actually changes in the context of CIRL that points AI towards human preferences any better then a straightforward reward learning setup? That’s the thing that I don’t think exists.\n", + "\n", + "**Daniel Filan:**\n", + "Okay, cool. So I guess on the other side of it, in terms of people who focus on optimality more than you might: in my imagination, there’s this reasoning that goes something like, look, when I think about super optimal things that AI systems could do in order to achieve some goal, as long as I can think of some property of an optimal system, it would be really surprising if whatever AI system that we train to do some goal can’t think of it, even if it’s only as competent as I am myself. And therefore, any thoughts I can generate about properties of optimal AI systems, there’s going to be some selection pressure for that. Just because, in order to get something that’s more competent than me at achieving goals, it’ll have this a similar sort of reasoning ability as me. I’m wondering, do you disagree with that, first of all, and if you do disagree with that, why? And if you don’t, maybe can you generate a stronger statement that you do disagree with?\n", + "\n", + "**Richard Ngo:**\n", + "So suppose we’re thinking about the first system that is better than humans at reasoning about optimality processes, and how they relate to AI alignment. It seems like our key goal should be making sure that system is using its knowledge in ways that are beneficial to us. So maybe telling us about it, or maybe using that knowledge to design a more align successor or something like that. But it seems very unclear the extent to which we can reason about that system itself, via talking about optimality constraints. It feels like my thinking here is about handing over to AI systems that are aligned enough that they can do the heavy lifting in the future. And I don’t think it’s necessarily a bad strategy to focus on just solving the whole problem in one go. But I do think that going about it via thinking in precise technical terms about the limit of intelligence seems a little bit off.\n", + "\n", + "**Daniel Filan:**\n", + "So I guess there’s this thought that’s like, maybe I can think of some strategy, something the limit of intelligence might take, or some idea that, oh , if you were really smart, you would try to treat humans instrumentally or something. So there’s some concern that if I can think of that, then almost by definition something smarter than me can think of it. And so it seems like there’s some instinct that says, look, once you have systems that are smarter than me, if these sorts of ideas actually like help that system do whatever it’s being selected to do, then it should use those ideas. Do you agree with that argument? Do you think that it doesn’t get you the kind of reasoning you regard as worrisome?\n", + "\n", + "**Richard Ngo:**\n", + "Well, it seems like the key question is whether those systems will in fact want to achieve goals that we’re unhappy about. It feels like the ways in which their motivations are shaped, whether that system will decide to do things that are bad for humans or not is going to be the type of thing that’s kind of shaped in a pretty messy way, a by a bunch of gradient descent on a bunch of data points. It’s kind of similar to the analogous human case: I am a human, and I’m reasoning about very intelligent future systems and trying to figure out which ones to instantiate, but the way in which I’m choosing which ones to instantiate is very dependent on gut emotional instincts that were honed over long period of evolution, childhood, and so on. Those emotions and instincts are things that feel very hard to reason about in a precise technical manner.\n", + "\n", + "**Daniel Filan:**\n", + "Okay. The idea is that sure, an AI system can figure out anything that I can think of. But maybe we have reason to think that we can have some sort of training process that would produce an AI that wanted to do stuff that the thing it’s supposed to be aligned to disapproves of. But actually, it’s just very unlikely to come up with an AI that has those sorts of desires and therefore, you don’t have to worry about this kind of reasoning.\n", + "\n", + "**Richard Ngo:**\n", + "That’s right.\n", + "\n", + "**Daniel Filan:**\n", + "Even though, if it did have those desires, it would do better on the outside objective. We just won’t train the overall objective to optimality.\n", + "\n", + "**Richard Ngo:**\n", + "Right. That’s, in some sense, the reason that we are not worried about dogs wanting to take over the world. Even though, we’ve done artificial selection on dogs for a while. In theory, at least, the signal that we are sending to them, as we do artificial selection, incentivizes them to take over the world, in some sort of stylized, abstract way. But in practice, that’s not the direction that their motivations are being pushed towards. We might hope that the same is true of systems, even if they’re a bit more intelligent or significantly more intelligent than humans. Where the extreme optima of whatever reward function we’re using is just not that relevant or salient.\n", + "\n", + "## The AI alignment research community \n", + "\n", + "### Updates from Yudkowsky conversation \n", + "\n", + "**Daniel Filan:**\n", + "You had this conversation with Eliezer Yudkowsky trying to formalize your disagreements, it seemed to me. I’m wondering if any of that conversation changed your mind and if you can say how it did?\n", + "\n", + "**Richard Ngo:**\n", + "I think, yeah. The two biggest things that I took away were number one, as I tried to explain my views about AI governance to Eliezer, I realized that they were missing a whole bunch of detail and nuance. And so any optimism that I have about AI governance needs to be grounded in much more specific details and plans for what might happen and so on. And that’s led to a bunch of recent work that I’m doing on formulating a bit more of a governance research agenda, and figuring out what the crucial considerations here are. So that was one thing that changed my mind, but that was a bit more about just like me trying to flesh out my own views.\n", + "\n", + "**Richard Ngo:**\n", + "I think in terms of Eliezer’s views, I think that I had previously underestimated how much his position relied on a few very deep abstractions that all fitted together. I don’t think you can really separate his views on intelligence, his views on consequentialism or agency, his views on recursive self-improvement, and things like that. You can look at different parts of it, but it seems like there’s this underlying deep-rooted set of intuitions that he keeps trying to explain in ways that people pick up on the particular thing he’s trying to explain, but without having a good handle on the overall set of intuitions. One particularly salient example of this is that previously, he talked a lot about utility functions, and then myself and Rohin Shah and a bunch of other people tried quite hard to figure out what specific argument he was making around utility functions. And we basically failed, because I don’t think that there’s a specific, precise, technical argument that you can make with our current understanding of utility theory that, that tells us about AGIs.\n", + "\n", + "**Richard Ngo:**\n", + "You don’t have a specific argument about utility functions and their relationship to AGIs in a precise, technical way. Instead, it’s more like utility functions are like a pointer towards the type of later theory that will give us a much more precise understanding of how to think about intelligence and agency and AGIs pursuing goals and so on. And to Eliezer, it seems like we’ve got a bunch of different handles on what the shape of this larger scale theory might look like, but he can’t really explain it in precise terms. It’s maybe in the same way that for any other scientific theory, before you latch onto it, you can only gesture towards a bunch of different intuitions that you have and be like, \"Hey guys, there are these links between them that I can’t make precise or rigorous or formal at this point.\"\n", + "\n", + "### Corrections to the community \n", + "\n", + "**Daniel Filan:**\n", + "Sort of related to updates: If there is one belief about existential risk from AI that you could create greater consensus about, among the population of people who are professional AI researchers, who thought carefully about existential risk from AI for more than 10 hours, what belief would it be?\n", + "\n", + "**Richard Ngo:**\n", + "Probably the main answer is just the thing I was saying before about how we want to be clear about where the work is being done in a specific alignment proposal. And it seems important to think about having something that doesn’t just shuffle the optimization pressure around, but really gives us some deeper reason to think that the problem is being solved. One example is when it comes to Paul Christiano’s work on amplification, I think one core insight that’s doing a lot of the work is that imitation can be very powerful without being equivalently dangerous. So yeah, this idea that instead of optimizing for a target, you can just optimize to be similar to humans, and that might still get you a very long way. And then another related insight that makes amplification promising is the idea that decomposing tasks can leverage human abilities in a powerful way.\n", + "\n", + "**Richard Ngo:**\n", + "Now, I don’t think that those are anywhere near complete ways of addressing the problem, but they gesture towards where the work is being done. Whereas for some other proposals, I don’t think there’s an equivalent story about what’s the deeper idea or principle that’s allowing the work to be done to solve this difficult problem. Maybe a second thing is that personally, in my more recent work about the alignment problem, I’ve been moving a little bit away from the term mesa-optimizers, or talking about a clean distinction between the outer alignment problem and the inner alignment problem. I think that it was an incredibly important idea when it first came out because it helped us clarify a bunch of confused intuitions about the relationship between reward functions and goals for intelligent systems, for example. But I think at this point, we might want to be thinking about what’s the spectrum between a system that’s purely guided by a reward function, versus a policy that has been trained on a reward function but now makes no reference to the reward function at all. I think these are two extremes and in practice, it seems unlikely that we’re going to end up at either extreme because reward functions are just very useful. So we should try and be thinking about what’s going on in the middle here and shaping our arguments accordingly.\n", + "\n", + "**Richard Ngo:**\n", + "Another way of thinking about that is, which is the correct analogy for reinforcement learning agents? Is it the case that gradient descent is like evolution, or is it the case that gradient descent is like learning in the human brain? And the answer is kind of a little bit of both. It’s going to play a role that’s intermediate between these two things, or has properties of both of these. And we shouldn’t treat these as two separate possibilities, but rather as two gestures towards what we should expect the future of AI to look like.\n", + "\n", + "**Daniel Filan:**\n", + "I think by now there’s some community of people who really think that AGI poses an existential risk, and that it’s really dangerous, and they’re doing research about it. There’s some intellectual edifice there. What do you think the strongest criticisms of that intellectual edifice around the idea of AGI risk are, that deserve more engagement from the community of researchers thinking about it?\n", + "\n", + "**Richard Ngo:**\n", + "I think the strongest criticisms used to be insufficient engagement with machine learning. And this has mostly been addressed these days. I think plausibly another criticism is that we, as a movement, probably haven’t been as clear as we could be in communicating about risks. So probably this is a slightly boring answer. I think there aren’t that many explanations, for example, of the AI alignment problem that are short, accessible, aimed at people who have competence with machine learning, and compelling. When you point people to Superintelligence, well, it doesn’t engage with machine learning. When you point people to something like Human Compatible, it actually doesn’t spend very much time on the reasons why we expect risks to arise. So I think that there’s some type of intellectual work here that just was nobody’s job for a while.\n", + "\n", + "**Richard Ngo:**\n", + "AGI Safety from First Principles was aimed at filling this gap. And then also more recent work by, for example, Joe Carlsmith from Open Philanthropy, with a report called \"Is Power Seeking AI an Existential Risk\" or something like that. But I still think that there’s a bit of a gap there, and I think filling that gap is profitable not just in terms of reaching out to other communities, but also just for having a better understanding of the problem, that will make our own research go better. I think that a lot of disagreements that initially seem like disagreements about which research is promising are actually more like disagreements about what the problem actually is in the first place. That’s kind of my standard answer.\n", + "\n", + "### Why others don’t join \n", + "\n", + "**Daniel Filan:**\n", + "Yeah. And I guess sort of as a converse of that question, you’ve worked at a few different AI research organizations with a bunch of people who are working on making AI more capable, and better at reasoning and pattern matching and such, but not particularly working on safety in terms of preventing existential risks. Why don’t you think that they have that focus?\n", + "\n", + "**Richard Ngo:**\n", + "It varies a bunch by person. I’d say a bunch of people are just less focused on AGI and more focused on pushing the field forward in a straightforward, incremental way. I think there’s a type of emotional orientation towards taking potentially very impactful arguments very seriously. And I think that a lot of people are quite cautious about those arguments, and to some extent rightfully so, because it’s very easy to be fooled by arguments that are very high level and abstract. So it’s almost like there’s this habit or there’s this predilection towards thinking, \"Man, this is really big if true.\" And then wanting to dig into it. And I think my guess is that probably the main thing blocking people from taking these ideas more seriously is just not getting that instinctive, \"Oh, wow, this is kind of crazy\" reaction, that makes them want to try very hard to figure it out for themselves. I think in his Most important Century series of blog posts, Holden Karnofsky does a really good job at addressing this intuition, just being like, \"This does sound crazy. And here are a bunch of outside view reasons to think that you should engage with the craziness. Here are a bunch of specific arguments on the object level. And here’s an emotional attitude that I take towards the problem.\" And yeah, that feels like it’s pretty directly aimed at the types of things that are determining a lot of machine learning researchers’ beliefs.\n", + "\n", + "## Richard Ngo as a researcher \n", + "\n", + "**Daniel Filan:**\n", + "Moving on, if somebody’s listening to this podcast, and they’re like, \"Richard Ngo seems like a productive researcher\", and they want to know what it’s like to be you doing research, what would you tell them? What’s your production function?\n", + "\n", + "**Richard Ngo:**\n", + "Right. I’ve been thinking a little bit about this lately because we’re also hiring for the team I’m leading at OpenAI. And what are the traits that I’m really excited about in a researcher? And I think it’s the combination of engaging with high level abstractions in a comfortable way, while also being very cautious about ways that those abstractions might break down or miss the mark. So I guess to me, it feels like a lot of what I’m doing with my research—I used the metaphor of bungee jumping recently—which is going as high as you can in terms of abstraction space, and then jumping down to try and get in touch with the ground again, and then trying to carry that ground level observation as far back up as you can. So that’s the sort of feeling that I have when I’m doing a lot of this conceptual research in particular. And then my personal style is to start very top down, and then be very careful to try and fill in enough details that things feel very grounded and credible. Was that roughly the thing you were aiming towards?\n", + "\n", + "## The world approaching AGI \n", + "\n", + "**Daniel Filan:**\n", + "Yeah, that seems roughly like an answer. And I guess the second to last question I’d like to ask: is there anything else that I should have asked you? And if so, what?\n", + "\n", + "**Richard Ngo:**\n", + "I feel like there’s something important to do with what the world looks like as we approach AGI. It feels like there’s a lot of work being done by either the assumption that the world will look roughly the same as it does now, except that these labs will be producing this incredibly powerful technology, or else the assumption that the world will be radically transformed by new applications floating around everywhere. And some of this has already been covered before. Maybe the thing that I’m really interested in right now is, what are the types of warning signs that actually make people sit up and pay attention as opposed to the types of warning signs that people just kind of dismiss with a bit of a shrug?\n", + "\n", + "**Richard Ngo:**\n", + "And I think COVID has been a particularly interesting case study, where you can imagine some worlds in which COVID is a very big warning sign that makes everyone pay attention to the risks of engineered pandemics. Or you can imagine a world in which people kind of collectively shrug and are like, okay, that was bad, and then don’t really think so much about like what might happen next. And it’s not totally clear to me which world we’re going to end up in, but it feels like somebody should be thinking hard about what are the levers that steer us towards this world or the other world. And then in the analogous case of AI, what are the levers that steer us from a given very impressive, large scale application of AI towards different possible narratives and different possible responses. And that’s something that I’m thinking about a bit more in my work today. It feels like an under-addressed question.\n", + "\n", + "## Following Richard’s work \n", + "\n", + "**Daniel Filan:**\n", + "The final thing I’d like to ask is, if somebody’s listening to this interview and they want to know more about you, or they maybe want to follow your research, how should they do so?\n", + "\n", + "**Richard Ngo:**\n", + "So the easiest way to follow my research is on the Alignment Forum, where I post most of the informal stuff that I’ve been working on. Another way you could follow my thinking in a higher bandwidth way is via Twitter, where I share work that I’m releasing as well as more miscellaneous thoughts. And you can find both of them by looking up my name, Richard Ngo. And then probably the other thing that I want to flag is the course that I’ve designed and which is currently running, which is the AGI Safety Fundamentals course. And this is my attempt to make the core ideas in AI alignment as accessible as possible, and have a curriculum people can work through and really grapple with the core issues. It’s run as a facilitated discussion group, so you go along every week for a couple of months and have discussions with a small group of people. We’ve been running that. This is the third cohort of people that we’re running now, and we’ll probably open up applications for another one in six months or so. So that’s something to keep an eye on. Or even if you don’t want to do the course itself, you can just have a look at the curriculum, which is, I think, a pretty good reading list for learning more about the field. We’ve also got a parallel program going on. One is on technical alignment, which I’ve been talking about. The other one is AI governance, and we’ve got curricula for both of those. So if you want to learn more, you can check those out. We’ve pretty carefully curated them to convey the core ideas in the field.\n", + "\n", + "**Daniel Filan:**\n", + "Great! Links to all of those will be in the show notes. So thanks for joining me today and to the listeners, I hope you got something out of this conversation.\n", + "\n", + "**Richard Ngo:**\n", + "Absolutely. Thanks a bunch, Daniel.\n", + "\n", + "**Daniel Filan:**\n", + "This episode is edited by Jack Garrett, and Justis Mills helped with transcription. The opening and closing themes are by Jack Garrett. The financial costs of making this episode are covered by a grant from the Long Term Future Fund. To read a transcript of this episode, or to learn how to support the podcast, you can visit axrp.net. Finally, if you have any feedback about this podcast, you can email me at feedback@axrp.net.\n", "\n" ] } @@ -618,39 +1148,43 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 40, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "40\n", - "{'total': 46, 'lesswrong': 37, 'alignment forum': 2, 'arxiv': 6, 'https://jsteinhardt.wordpress.com': 1}\n", - "223697\n", - "179\n" + "Articles count: 150\n", + "Num of each source: {'alignment forum': 105, 'manual': 0, 'waitbutwhy.com': 0, 'https://aiimpacts.org': 11, 'arbital.com': 10, 'https://intelligence.org': 22, 'reports': 2, 'https://aisafety.camp': 0, 'curriculum': 0, 'https://www.yudkowsky.net': 2, 'distill': 0, 'total': 152}\n", + "Num chars: 1893590\n", + "Num words: 306879\n", + "Num sentences: 13750\n", + "Num blocks: 1434\n" ] } ], "source": [ - "print(len(dataset.data))\n", - "print(dataset.num_articles)\n", - "print(dataset.total_char_count)\n", - "print(len(dataset.embed_split))" + "print(f\"Articles count: {len(dataset.data)}\")\n", + "print(f\"Num of each source: {dataset.num_articles}\")\n", + "print(f\"Num chars: {dataset.total_char_count}\")\n", + "print(f\"Num words: {dataset.total_word_count}\")\n", + "print(f\"Num sentences: {dataset.total_sentence_count}\")\n", + "print(f\"Num blocks: {dataset.total_block_count}\")" ] }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Completed 50/179\n", - "Completed 100/179\n", - "Completed 150/179\n" + "Completed 50/197\n", + "Completed 100/197\n", + "Completed 150/197\n" ] } ], @@ -662,306 +1196,1472 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "I believe this may be a good addition to the cognitive bias literature:\n", + "Glossary of Qualia Research Institute Terms This is a glossary of key terms and concept handles that are part of the memetic ecosystem of the Qualia Research Institute. Reading this glossary is itself a great way to become acquainted with this emerging memeplex. If you do not know what a memeplex is… you can find its definition in this glossary. Basics Consciousness (standard psychology, neuroscience, and philosophy term): There are over a dozen common uses for the word consciousness, and all of them are interesting. Common senses include: self-awareness, linguistic cognition, and the ability to navigate one’s environment. With that said, the sense of the word in the context of QRI is more often than not: the very fact of experience, that experience exists and there is something that it feels like to be. Talking loosely and evocatively- rather than formally and precisely- consciousness refers to “what experience is made of”. Of course formalizing that statement requires a lot of unpacking about the nature of matter, time, selfhood, and so on. But this is a start. Qualia (standard psychology, neuroscience, and philosophy term): This word refers to the range of ways in which experience presents itself. Experiences can be richly colored or bare and monochromatic, they can be spatial and kinesthetic or devoid of geometry and directions, they can be flavorfully blended or felt as coming from mutually unintelligible dimensions, and so on.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", "\n", + "Classic qualia examples include things like the redness of red, the tartness of lime, and the glow of bodily warmth. However, qualia extends into categories far beyond the classic examples, beyond the wildest of our common-sense conceptions. There are modes of experience as altogether different from everything we have ever experienced as vision qualia is different from sound qualia. Valence / Hedonic Tone (standard psychology, neuroscience, and philosophy term): How good or bad an experience feels – each experience expresses a balance between positive, neutral, and negative notes. The aspect of experience that accounts for its pleasant and unpleasant qualities. The term is evocative of pleasant sensations such as warming up one’s body when cold with a blanket and a cup of hot chocolate. That said, hedonic tone refers to a much broader class of sensations than just the feeling of warmth. For example, the music appreciation enhancement produced by drugs can be described as “enhanced hedonic tone in sound qualia”. Hedonic tone can appear in any sensory modality (touch, smell, sight, etc. ), and even more generally, in every facet of experience (such as cognitive and proprioceptive elements, themselves capable of coming with their own flavor of euphoria/dysphoria). Experiences with both negative and positive notes are called “mixed”, which are the most common ones.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", "\n", - "# > Following the Crowd: Brain Substrates of Long-Term Memory Conformity\n", + "Helpful Philosophy Ontology (standard high-level philosophy term; ref: 1): At the most basic level, an ontology is an account of what is real and what is good. Epistemology (standard high-level philosophy term; ref: 1): The set of strategies, heuristics, and methods for knowing. In the context of consciousness research, what constitutes a good epistemology is a highly contentious subject. Some scientists argue that we should only take into account objectively-measurable third-person data in order to build models and postulate theories about consciousness (cf. heterophenomenology). On the other extreme, some argue that the only information that counts is first-person experiences and what they reveal to us (cf. new mysterianism). Somewhere in the middle, QRI fully embraces objective third-person data. And along with it, QRI recognizes the importance of skepticism and epistemic rigor when it comes to which first-person accounts should be taken seriously. Its epistemology does accept the information gained from alien state-spaces of consciousness as long as they meet some criteria. For example, we are very careful to distinguish between information about the intentional content of experience (what it was about) and information about its phenomenal character (how it felt).\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", "\n", + "As a general heuristic, QRI tends to value more e.g. trip reports that emphasize the phenomenal character of the experience (e.g. “30Hz flashes with slow-decay harmonic reverb audio hallucinations”) relative to intentional content (e.g. “the DMT alien said I should learn to play the guitar”). Ultimately, first-person and third-person data are complementary views of the same substrate of consciousness (cf. dual-aspect monism), and so are both equally necessary for a complete scientific account of consciousness. Functionalism (standard high-level philosophy term; ref: 1, 2): In Philosophy of Mind, functionalism is the view that consciousness is produced (and in some cases identical with) not only by the input-output mapping of an information-processing system, but also by the internal relationships that make that information-processing possible. In light of Marr’s Levels of Analysis (see below), we could say that functionalism identifies the content of conscious experience with the algorithmic level of analysis. Hence this philosophy is usually presented in conjunction with the concept of “substrate neutrality” which posits that the material makeup of brains is not necessary for the arising of consciousness out of it.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", "\n", - " - Micah Edelson1,*, \n", + "If we implemented the same information-processing functions that are encoded in the neural networks of a brain using rocks, buckets of water, or a large crowd instantiating a large computer, we would also generate the same experiences the brain generates on its own. Importantly, functionalism tends to deny any essential role of the substrate in the generation of consciousness, and will typically also deny any significant interaction between levels of analysis (see below). Eliminativism (standard high-level philosophy term; ref: 1, 2, 3): In Philosophy of Mind, eliminativism refers to a cluster of ideas concerning whether the word “consciousness” is clear enough to be useful for making sense of how brains work. One key idea in eliminativist views is that most of the language that we use to talk about experiences (from specific emotions to qualia) is built on top of folk-psychology rather than physical reality. In a way, terms such as “experience” and “feelings” are an interface for the brain to model itself and others in a massively simplified but adaptive way. There is no reason why our evolved intuitions about how the brain works should even approximate how it really works. In many cases, eliminativists advocate starting from scratch and abandoning our intuitions about experience, sticking to hard physical and computational analysis of the brain as empirically measured.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", "\n", - " - Tali Sharot2, \n", - "\n", - " - Raymond J. Dolan2, \n", - "\n", - " - Yadin Dudai1\n", - "\n", - "1Department of Neurobiology, Weizmann Institute of Science, Israel.\n", - "\n", - "\n", - " - 2Wellcome Trust Centre for Neuroimaging, Institute of Neurology, University College London, London, UK.\n", - "\n", - "## ABSTRACT\n", - "\n", - "Human memory is strikingly susceptible to social influences, yet we know little about the underlying mechanisms. We examined how socially induced memory errors are generated in the brain by studying the memory of individuals exposed to recollections of others. Participants exhibited a strong tendency to conform to erroneous recollections of the group, producing both long-lasting and temporary errors, even when their initial memory was strong and accurate. Functional brain imaging revealed that social influence modified the neuronal representation of memory. Specifically, a particular brain signature of enhanced amygdala activity and enhanced amygdala-hippocampus connectivity predicted long-lasting but not temporary memory alterations. Our findings reveal how social manipulation can alter memory and extend the known functions of the amygdala to encompass socially mediated memory distortions.\n", - " - Title: A study in Science on memory conformity, Author: dvasya, Date published: 2011-07-15, URL: https://www.lesswrong.com/posts/fQE3cP9NjS5sLm8wr/a-study-in-science-on-memory-conformity\n", - "\n", - "\n", - "http://​​www.sciencemag.org/​​content/​​333/​​6038/​​108.full\n", - "\n", - "http://​​ifile.it/​​v76wsi5\n", - "\n", - " - Title: A study in Science on memory conformity, Author: dvasya, Date published: 2011-07-15, URL: https://www.lesswrong.com/posts/fQE3cP9NjS5sLm8wr/a-study-in-science-on-memory-conformity\n", - "\n", - "## Design sketches\n", - "\n", - "1. Notes on differences between these images and what we’re planning on implementing:\n", - "\n", - "\n", - "\n", - "2. Meetup widget in sidebar, listing the 5 nearest meetups occurring in the next 2 weeks, linking to:(something like right-click & \"Open Image in new tab\" will help here)\n", + "This view suggests that once we truly understand scientifically how brains work, the language we will use to talk about it will look nothing like the way we currently speak about our experiences, and that this change will be so dramatic that we would effectively start thinking as if “consciousness never existed to begin with”.\n", "\n", "\n", "\n", "\n", + "Presentism (standard high-level philosophy term; ref: 1): The view that only the present is real, the past and the future being illusory inferences and projections made in the present. Oftentimes presentism posits that change is a fundamental aspect of the present and that the feeling of the passage of time is based on the ever-changing nature of reality itself.\n", + "Eternalism (standard high-level philosophy term; ref: 1): The view that every here-and-now in reality is equally real. Rather than thinking of the universe as a “now” sandwiched between a “past” and “future”, eternalism posits that it is more accurate to simply describe pairs of moments as having a “before” and “after” relationship, but neither of them being in the future or past. Some of the strongest arguments for eternalism come from Special and General Relativity (see: Rietdijk–Putnam argument), where space-time forms a continuous 4-dimensional geometric shape that stands together as a whole, and where any notion of a “present” is only locally valid. In some sense, eternalism says that all of reality exists in an “eternal now” (including your present, past, and future selves).\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", "\n", - "3. New headers:\n", + " Personal Identity (standard high-level philosophy term; ref: 1): The relevant sense of this term for our purposes refers to the set of questions about what constitutes the natural unit for subjects of experience. Questions such as “will the consciousness who wakes up in my current body tomorrow morning be me? ”, “if we make an atom-by-atom identical copy of me right now, will I start existing in it as well? ”, “if you conduct a Wada Test, is the consciousness generated by my right hemisphere alone also me? ”, and so on. Closed Individualism (coined by Daniel Kolak; ref: 1): In its most basic form, this is the common-sense personal identity view that you start existing when you are born and stop existing when you die. According to this view each person is a different subject of experience with an independent existence. One can believe in a soul ontology and be a Closed Individualist at the same time, with the correction that you exist as long as your soul exists, which could be the case even before or after death. Empty Individualism (coined by Daniel Kolak; ref: 1, 2, 3): This personal identity view states that each “moment of experience” is its own separate subject. While it may seem that we exist as persons with an existence that spans decades, Empty Individualism does not associate a single subject to each person.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Rather, each moment a new “self” is born and dies, existing for as long as the conscious event takes place (something that could be anywhere between a femtosecond and a few hundred milliseconds, depending on which scientific theory of consciousness one believes in). Open Individualism (coined by Daniel Kolak; ref: 1, 2, 3, 4): This is the personal identity view that we are all one single consciousness. The apparent partitions and separations between the universal consciousness, in this view, are the result of partial information access from one moment of experience to the next. Regardless, the subject who gets to experience every moment is the same. Each sentient being is fundamentally part of the same universal subject of experience. Goldilocks Zone of Oneness (QRI term; 1, 2, 3): Having realized that there are both positive and negative psychological aspects to each of the three views of personal identity discussed (Closed, Empty, Open Individualism), the Goldilocks Zone of Oneness emerges as a conceptual resolution. Open Individualism comes with a solution to the fear of death, but it also can give rise to a sort of cosmic solipsism. Closed Individualism allows you to feel fundamentally special, but also disconnected from the universe and fundamentally misunderstood by others. Empty Individualism is philosophically satisfying, but it may come with a sense of lack of agency and the fear of being a time-slice that is stuck in a negative place.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "The Goldilocks Zone of Oneness posits that there is a way to transcend classical logic in personal identity, and that the truth incorporates elements of all of the three views at once. In the Goldilocks Zone of Oneness one is simultaneously part of a whole but also not the entirety of it. One can relate with others by having a shared nature, while also being able to love them on their own terms by recognizing their unique identity. This view has yet to be formalized, but in the meantime it may prove to be pragmatically useful for community-building. The Problem of Other Minds (standard high-level philosophy term; ref: 1, 2): This is the philosophical conundrum of whether other people (and sentient beings in general) are conscious. While your own consciousness is self-evidence, the consciousness of others is inferred. Possible solutions involve technologies such as the Generalized Wada Test (see below), phenomenal puzzles, and thalamic bridges, which you can use to test the consciousness of another being by having it solve a problem that can only be solved by making comparisons between qualia values. Solipsism (standard high-level philosophy term; ref: 1, 2, 3): In its classic formulation, solipsism refers to a state of existence in which the only person who is conscious is “oneself”, which resides in the body of an individual human over time. A more general version of solipsism involves crossing it with personal identity views (see above).\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Through this lens, the classic person-centric formulation of solipsism refers exclusively to a Closed Individualist universe. Alternatively, Open Individualism also has a solipsistic interpretation – it is thus compatible with (and in at least in one sense entails) solipsism: the entire multiverse of experiences are all experiences of a single solipsistic cosmic consciousness. With an Empty Individualist universe, too, we can have a solipsistic interpretation of reality. In one version you use epiphenomenalism to claim that this moment of experience is the only one that is conscious even though the whole universe still exists and it had an evolutionary path that led it to the configuration in which you stand right now. In another version, one’s experience is the result of the fact that in the cosmic void everything can happen. This is not because it is likely, but because there is a boundless amount of time for it to happen. That is, no matter how thin its probability is, it will still take place at some point (see: Boltzmann brain). That said, one’s present experience -with its highly specific information content- being the only one that exists seems very improbable a priori. Like imagining that despite the fact that “the void can give rise to anything” the only thing that actually gets materialized is an elephant. Why would it only produce an elephant, of all things? Likewise, solipsistic Empty Individualism has this problem – why would this experience be the only one?\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "To cap it off, we can also reason about solipsism in its relation to hybrid views of personal identity. In their case solipsism either fails, or its formulation needs to be complicated significantly. This is partly why the concept of the Goldilocks Zone of Oneness (see above) might be worth exploring, as it may be a way out of ultimate solipsism. On a much more proximal domain, it may be possible to use Phenomenal Puzzles, Wada tests, and ultimately mindmelding to test the classical (Closed Individualist) formulation of solipsism. Suffering Focused Ethics (recent philosophy term from rationalist-adjacent communities; ref: 1, 2) The view that our overriding obligation is to focus on suffering. In particular, taking seriously the prevention of extreme suffering is one of the features of this view. This is not unreasonable if we take into account the logarithmic scales of pain and pleasure into account, which suggest that the majority of suffering is concentrated in a small percent of experiences of intense suffering. Hence why caring about the extreme cases matters so much. Antinatalism (standard high-level philosophy term; ref: 1, 2): This is the view that being born entails a net negative. Classic formulations of this view tend to implicitly assume Closed Individualism, where there is someone who may or may not be born and it is meaningful to consider this a yes or no question with ontological bearings.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Under Open Individualism the question becomes whether there should be any conscious being at all, for neither preventing someone’s birth nor committing an individual suicide entail the real birth or death of a consciousness. They would merely add or subtract from the long library corridors of experiences had by universal consciousness. And in Empty Individualism, antinatalism might be seen through the light of “preventing specific experiences with certain qualities”. For example, having an experience of extreme suffering is not harming a person (though it may have further psychological repercussions), but rather harming that very experience in an intrinsic way. This view would underscore the importance of preventing the existence of experiences of intense suffering rather than preventing the existence of people as such. A final note on antinalism is that even in its original formulation we encounter the problem that selection pressures makes any trait that reduces inclusive fitness disappear in the long run. The traits that predispose to such views would simply be selected out. A more fruitful way of improving the world is to encourage the elimination of suffering in ways that do not reduce inclusive fitness, such as the prevention of genetic spell errors and diseases that carry a high burden of suffering.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Tyranny of the Intentional Object (coined by David Pearce; ref: 1, 2): The way our reward architecture is constructed makes it difficult for us to have a clear sense of what it is that we enjoy about life. Our brains reinforce the pursuit of specific objects, situations, and headspaces, which gives the impression that these are intrinsically valuable. But this is an illusion. In reality such conditions trigger positive valence changes to our experience, and it is those that we are really after (as evidenced by the way in which our reward architecture is modified in presence of euphoric and dysphoric drugs and external stimuli such as music). We call this illusion the tyranny of the intentional object because in philosophy “intentionality” refers to “what the experience is about”. Our world-simulations chain us to the feeling that external objects, circumstances, and headspaces are the very source of value. More so, dissociating from such sources of positive valence triggers negative valence, so critical insight into the way our reward architecture really works is itself negatively reinforced by it. Formalism Terms Formalism (standard high-level philosophy term; ref: 1, 2): Formalism is a philosophical and methodological approach for analyzing systems which postulates the existence of mathematical objects such that their mathematical features are isomorphic to the properties of the system.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "An example of a successful formalism is the use of Maxwell’s equations in order to describe electromagnetic phenomena. Qualia Formalism (QRI term; 1, 2, 3): Qualia Formalism means that for any given physical system that is conscious, there will be a corresponding mathematical object associated to it such that the mathematical features of that object will be isomorphic to the phenomenology of the experience generated by the system. Marr’s Levels of Analysis (standard cognitive science term; ref: 1, 2): This powerful analytic framework was developed by cognitive scientist David Marr to talk more precisely about vision, but it is more broadly applicable to information processing systems in general. It is a way to break down what a system does in a conceptually clear fashion that lends itself to a clean analysis. Computational Level (standard cognitive science term; ref: 1, 2): The first of three of Marr’s Levels of Analysis, the Computational Level of abstraction describes what the system does from a third-person point of view. That is, the input-output mapping, the runtime complexity for the problems it can solve, and the ways in which it fails are all facts about a system that are at the computational level of abstraction. In a simple example case, we can describe an abacus at the computational level by saying that it can do sums, subtractions, multiplications, divisions, and other arithmetic operations.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Algorithmic Level (standard cognitive science term; ref: 1, 2): The second of three of Marr’s Levels of Analysis, the Algorithmic Level of abstraction describes the internal representations, operations, and their interactions used to transform the input into the output. In aggregate, representations, operations, and their interactions constitute the algorithms of the system. As a general rule, we find that there are many possible algorithms that give rise to the same computational-level properties. Following the simple example case of an abacus, the algorithmic-level account would describe how passing beads from one side to another and using each row to represent different orders of magnitude are used to instantiate algorithms to perform arithmetic operations. Implementation Level (standard cognitive science term; ref: 1, 2): The third of three of Marr’s Levels of Analysis, the Implementation Level of abstraction describes the way in which the system’s algorithms are physically instantiated. Following the case of the abacus, an implementation-level account would detail how the various materials of the abacus are put together in order to allow the smooth passing of beads between the sides of each row and how to prevent them from sliding by accident (and “forgetting” the state). Interaction Between Levels (obscure cognitive science concept handle; ref: 1, 2): Some information-processing systems can be fully understood by describing each of Marr’s Levels of Analysis separately.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "For example, it does not matter whether an abacus is made of metal, wood, or even if it is digitally simulated in order to explain its algorithmic and computational-level properties. But while this is true for an abacus, it is not the case for analog systems that leverage the unique physical properties of their components to do computational shortcuts. In particular, in quantum computing one intrinsically requires an understanding of the implementation-level properties of the system in order to explain the algorithms used. Hence, for quantum computing, there are strong interactions between levels of analysis. Likewise, we believe this is likely going to be the case for the algorithms our brains perform by leveraging the unique properties of qualia. Natural Kind (standard high-level philosophy term; ref: 1, 2): Natural kinds are things whose objective existence makes it possible to discover durable facts about them. They are the elements of a “true ontology” for the universe, and what “carves reality at its joints”. This is in contrast to “reifications” which are aggregates of elements with no unitary independent existence. State-Space (standard term in physics and mathematics; ref: 1, 2): A state-space of a system is a geometric map where each point corresponds to a particular state of the system.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Usually the space has a Euclidean geometry with a number of dimensions equal to the number of variables in the system, so that the value of each variable is encoded in the value of a corresponding dimension. This is not always the case, however. In the general case, not all points in the state-space are physically realizable. Additionally, some system configurations do not admit a natural decomposition into a constant set of variables. This may give rise to irregularities in the state-space, such as non-Euclidean regions or a variable number of dimensions. State-Space of Consciousness (coined by David Pearce; 1, 2, 3): This is a hypothetical map that contains the set of all possible experiences, organized in such a way that the similarities between experiences are encoded in the geometry of the state-space. For example, the experience you are having right now would correspond to a single point in the state-space of consciousness, with the neighboring experiences being Just Noticeably Different from your experience right now (e.g. simplistically, we could say they would be different from your current experience “by a single pixel”). Qualia Value (QRI term; ref: 1): Starting with examples-  the scent of cinnamon, a spark of sourness, a specific color hue, etc. are all qualia values. Any particular quality of experience that cannot be decomposed further into overlapping components is a qualia value.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Qualia Variety (QRI term; ref: 1): A qualia variety refers to the set of qualia values that belong to the same category (for example, tentatively, phenomenal colors are all part of the same qualia variety, which is different from the qualia variety of phenomenal sounds). A possible operationalization for qualia varieties involves the construction of equivalence classes based on the ability to transform a given qualia value into another via a series of Just-Noticeable Differences. For example, in the case of color, we can transform a given qualia value like a specific shade of blue, into another qualia value like a shade of green by traversing across a straight line from one to the other in the CIELAB color space. Tentatively, it is not possible to do the same between a shade of blue and a particular phenomenal sound. That said, the large number of unknowns (and unknown unknowns! ) about the state-space of consciousness does not allow us to rule out the existence of qualia values that can bridge the gap between color and sound qualia. If that turned out to be the case, we would need to rethink our approach to defining qualia varieties. Region of the State-Space of Consciousness (QRI term; ref: 1, 2): A set of possible experiences that are similar to each other in some way. Given an experience, the “experiences nearby in the state-space of consciousness” are those that share its qualities to a large degree but have variations.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "The term can be used to point at experiences with a given property (such as “high-valence” and “phenomenal color”). The Binding Problem (standard psychology, neuroscience, and philosophy term; ref: 1, 2): The binding problem (also called the combination problem) arises from asking the question: how is it possible that the activity of a hundred billion neurons that are spatially distributed can simultaneously contribute to a unitary moment of experience? It should be noted that in the classical formulation of the problem we start with an “atomistic” ontology where the universe is made of space, particles, and forces, and the question then becomes how spatially-distributed discrete particles can “collaborate” to form a unified experience. But if one starts out with a “globalistic” ontology where the universe is made of a universal wavefunction, then the question that arises is how something that is fundamentally unitary (the whole universe) can give rise to “separate parts” such as individual experiences, which is often called “the boundary problem”. Thus, the “binding problem” and “the boundary problem” are really the same problem, but starting with different ontologies (atomistic vs. globalistic).\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Phenomenal Binding (standard high-level philosophy term; ref: 1, 2): This term refers to the hypothetical mechanism of action that enables information that is spatially-distributed across a brain (and more generally, a conscious system) to simultaneously contribute to a unitary discrete moment of experience. Local Binding (lesser-known cognitive science term; ref: 1): Local binding refers to the way in which the features of our experience are interrelated. Imagine you are looking at a sheet of paper with a drawing of a blue square and a yellow triangle. If your visual system works well you do not question which shape is colored blue; the color and the shapes come unified within one’s experience. In this case, we would say that color qualia and shape qualia are locally bound. Disorders of perception show that this is not always the case: people with simultagnosia find it hard to perceive more than one phenomenal object at a time and thus would confuse the association between the colors and shapes they are not directly attending to, people with schizophrenia have local binding problems in the construction of their sense of self, and people with motion blindness have a failure of local binding between sensory stimuli separated by physical time. Global Binding (lesser-known cognitive science term; ref: 1, 2): Global binding refers to the fact that the entirety of the contents of each experience is simultaneously apprehended by a unitary experiential self.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "As in the example for local binding, while blue and the square (and the yellow and the triangle) are locally bound into separate phenomenal objects, both the blue square and the yellow triangle are globally bound into the same experience. The Mathematics of Valence\n", + "Valence Realism (QRI term; ref: 1): This is the claim that valence is a crisp phenomenon of conscious states upon which we can apply a measure. Also defined as: “Valence (subjective pleasantness) is a well-defined and ordered property of conscious systems.”\n", + "Valence Structuralism (QRI term; ref: 1): Valence could have a simple encoding in the mathematical representation of a system’s qualia. Symmetry Theory of Valence (QRI term; 1, 2, 3): Given a mathematical object isomorphic to the qualia of a system, the mathematical property which corresponds to how pleasant it is to be that system is that object’s symmetry. Valence Gradients (QRI term; ref: 1, 2): It is postulated that one of the important inputs that contributes to our decision-making involves “valence gradients”. To understand what a valence gradient is, it is helpful to provide an example. Imagine coming back from dancing in the rain and feeling pretty cold. In order to warm yourself up you get into the shower and turn on the hot water. Ouch! Too hot, so you dial down the temperature. Brrr! Now it’s too cold, so you dial up the temperature just a little. Ah, just perfect!\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "See, during this process you evaluated, at each point, in what way you could modify your experience in order to make it feel better. At first the valence gradient was pointing in the direction of higher temperature. As soon as you felt it being too hot, the valence gradient changed direction and pointed to lower temperature. And so on until it feels like there is nothing else you could do to improve how you feel. In the more general case, we posit that a significant input into our decision-making is the direction of change along which we believe our experience would improve. At an implementation level of analysis (see above) the very syntax of our experience might be built with a landscape of valence gradients. In a sense, noticing them is possible, but it is a task akin to the metaphor of a fish not knowing what water is. We use valence gradients to navigate both the external and internal world in such a basic and all-pervasive way that missing this fact altogether is easy. When we justify why we did such and such, we often forget that a big component of the decision was made based on how each of the options felt. The difficulty we face when trying to point at the specific valence gradients that influence our decision-making is one of the reasons why the tyranny of the intentional object (see above) arises, which is that what pulls and pushes us is not explicitly represented in our conceptual scheme. Click to view slideshow.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "CDNS Analysis (QRI term; ref: 1, 2): A scientific and philosophical hypothesis with implications for measuring valence in conscious systems. Namely, the hypothesis is that the Symmetry Theory of Valence is expressed in the structure of neural patterns over time, implying that the valence of a brain will be in part determined by neural dissonance, consonance, and noise. This makes precise, empirically testable predictions within paradigms such as Connectome-Specific Harmonic Waves. Research Paradigms Evolutionary Qualia (QRI term): Evolutionary Qualia is a scientific discipline that will emerge as the science of consciousness improves to the point where cellular gene expression analysis, brain imaging, and interpretation algorithms get to infer the qualia present in the experience of the brains of animals in general. For instance, we may find out that certain combinations of receptor types and protein shapes inside neurons of the visual cortex are necessary and sufficient for generating color qualia. Additionally, such understanding could be complemented with an information-theoretic account of why color qualia is more effective (cost-benefit-wise) for certain information-processing than other qualia. Together, these two kinds of understanding will allow us to explain why the specific qualia that we have was recruited by natural selection for information-processing purposes.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Evolutionary Qualia is the (future) discipline that explains from an evolutionary point of view why we have the specific qualia and patterns of local binding that we do (said differently, it will explain why “the walls of our world-simulation are painted the way they are”). So while Evolutionary Psychology may explain why we have evolved to have some emotions from the point of view of their behavioral effects, Evolutionary Qualia will explain why the emotions feel the way they do and how those specific feelings happen to have the right “shape” for the information-processing tasks they accomplish. Algorithmic Reduction (QRI term; ref: 1, 2): A reduction is a model that explains a set of behaviors, often very complex and diverse, in terms of the interaction between variables. A successful reduction is one that explains the intricacies and complexities present in the set of behaviors as emergent effects from a much smaller number of variables and their interactions. A specific case is that of “atomistic reductions” which decompose a set of behaviors in terms of particles interacting with each other (e.g. ideal gas laws from statistical mechanics in physics). While many scientifically significant reductions are atomistic in nature, one should not think that every phenomenon can be successfully reduced atomistically (e.g. double-slit experiment). Even when a set of behaviors cannot be reduced atomistically we may be able to algorithmically reduce it.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "That is, to identify a set of processes, internal representations, and interactions that when combined give rise to the set of observed behaviors. This style of reduction is very useful in the field of phenomenology since it can provide insights into how complex phenomena (such as psychedelic hallucinations) emerge out of a few relatively simple algorithmic building blocks. This way we avoid begging the question by not assuming an atomistic ontology in a context where it is not clear what atoms correspond to. Psychedelic Cryptography (QRI term; ref: 1, 2, 3): Encoding information in videos, text, abstract paintings, etc. such that only people who are in a specific state of consciousness can decode it. A simple example is the use of alternations in after-image formation on psychedelics (enhanced persistence of vision, aka. tracers) to paint a picture by presenting the content of an image one column of pixels at a time. Sober individuals only see a column of pixels while people high on psychedelics will see a long trace forming parts of an image that can be inferred by paying close attention. In general, psychedelic cryptography can be done by taking advantage of any of the algorithms one finds with algorithmic reductions of arbitrary states of consciousness. In the case of psychedelics, important effects that can be leveraged include tracers, pareidolia, drifting, and symmetrification.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Psychedelic Turk (QRI term; ref: 1, 2, 3, 4): Mechanical Turk is a human task completion platform that matches people who need humans to do many small (relatively) easy tasks with humans willing to do a lot of small (relatively) easy tasks. Psychedelic Turk is akin to Mechanical Turk, but where workers disclose the state of consciousness they are in. This would be helpful for task requesters because many tasks are more appropriate for people in specific states of consciousness. For example, it is better to test ads intended to be seen by drunk people by having people who are actually drunk evaluate them, as opposed to asking sober people to imagine how they would perceive them while drunk. Likewise, some high-stakes tasks would benefit from being completed by people who are demonstrably very alert and clear-headed. And for foundational consciousness research, Psychedelic Turk would be extremely useful as it would allow researchers to test how people high on psychedelics and other exotic agents process information and experience emotions usually inaccessible in sober states. Generalized Wada Test (QRI term; ref: 1, 2, 3): This is a generalization of the Wada Test where rather than pentobarbital being injected in just one hemisphere while the other hemisphere is kept sober, one injects substance A in one hemisphere and substance B on the other. This could be used to improve our epistemology about various states of consciousness.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "By keeping one hemisphere in a state with robust linguistic ability the other hemisphere could be used to explore alien-state spaces of consciousness and allow for real-time verbal interpretation. The caveats and complications are myriad, but the general direction this concept handle is pointing to is worth exploring. Phenomenology Self-Locating Uncertainty (originally a physics term but we also use it for describing a phenomenal character of experience; ref: 1, 2): The uncertainty that one has about who and where one is. This is relevant in light of states of consciousness that are common on high-dose psychedelics, mental illnesses, and meditation, where the information about one’s identity and one’s place in the world is temporarily inaccessible. Very high- and low-valence states tend to induce a high level of self-locating uncertainty as the information content of the experience is over-written by very simple patterns that dominate one’s attention. Learning to navigate states with self-locating uncertainty without freaking out is a prerequisite for studying alien state-spaces of consciousness. Phenomenal Time (standard high-level philosophy term; ref: 1): The felt-sense of the passage of time. This is in contrast to the physical passage of time. Although physical time and phenomenal time tend to be intimately correlated, as you will see in the definition of “exotic phenomenal time” this is not always the case.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Phenomenal Space (standard high-level philosophy term; ref: 1, 2): The experience of space. Usually our sense of space represents a smooth 3D Euclidean space in a projective fashion (with variable scale encoding subjective distance). In altered states of consciousness phenomenal space can be distorted, expanded, contracted, higher-dimensional, topologically distinct, and even geometrically modified as in the case of hyperbolic geometry while on DMT (see below). Pseudo-Time Arrow (QRI term; ref: 1): This is a formal model of phenomenal time. It utilizes a simple mathematical object: a graph. The nodes of the graph are identified with simple qualia values (such as colors, basic sounds, etc. ) and the edges are identified with local binding connections. According to the pseudo-time arrow model, phenomenal time is isomorphic to the patterns of implicit causality in the graph, as derived from patterns of conditional statistical independence. Exotic Phenomenal Time (QRI term; ref: 1): It is commonly acknowledged that in some situations time can feel like it is passing faster or slower than normal (cf. tachypsychia). What is less generally known is that experiences of time can be much more general, such as feeling like time stops entirely or that one is stuck in a loop. These are called exotic phenomenal time experiences, and while not very common, they certainly are informative about what phenomenal time is.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Deviations from an apparent universal pattern are usually scientifically significant. Reversed Time (QRI term; ref: 1): This is a variant of exotic phenomenal time in which experience seems to be moving backwards in time. “Inverted tracers” are experienced where one first experiences the faint after-images of objects before they fade in, constitute themselves, and then quickly disappear without a trace. According to the pseudo-time arrow model this experience can be described as an inversion of the implicit arrow of causality, though how this arises dynamically is still a mystery. Moments of Eternity (common psychedelic phenomenology term; ref: 1): This exotic phenomenal time describes experiences where all apparent temporal movement seems to stop. One’s experience seems to have an unchanging quality and there is no way to tell if there will ever be something else other than the present experience in the whole of existence. In most cases this state is accompanied by intense emotions of simple texture and immediacy (rather than complex layered constructions of feelings). The experience seems to appear as the end-point and local maxima of annealing on psychedelic and dissociative states. That is, it often comes as metastable “flashes of large-scale synchrony” that are created over the course of seconds to minutes and decay just as quickly. Significantly, sensory deprivation conditions are ideal for the generation of this particular exotic phenomenal time.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Timelessness (QRI term; ref: 1): Timelessness is a variant of exotic phenomenal time where causality flows in a very chaotic way at all scales. This prevents forming a general global direction for time. In the state, change is perceptible and it is happening everywhere in your experience, and yet it seems as if there is no consensus among the different parts of your experience about the direction of time. That is, there is no general direction along which the experience seems to be changing as a whole over time. The chaotic bustle of changes that make up the texture of the experience are devoid of a story arc, and yet remain alive and turbulent. Trip reports suggest that the state that arises at the transition points between dissociative plateaus has this noisy timelessness quality (e.g. coming up on ketamine). Listening to green noise evokes the general idea, but you need to imagine that happening on every sensory modality and not just audio. Time Loops (common psychedelic phenomenology term; ref: 1): This is perhaps the most common exotic phenomenal time experience that people have on psychedelics and dissociatives. This is due to the fact that, while it can be generated spontaneously, it is relatively easy to trigger by listening to repetitive music (e.g. a lot of EDM, trance, progressive rock, etc.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "), repetitive movements (e.g. walking, dancing), and repetitive thoughts (e.g. talking about the same topic for a long time) all of which are often abundant in the set and setting of psychedelic users. The effect happens when your projections about the future and the past are entirely informed by what seems like an endlessly repeating loop of experience. This often comes with intense emotions of its own (which are unusual and outside of the normal range of human experience), but it also triggers secondary emotions (which are just normal emotions amplified) such as fear and worry, or at times wonder and bliss. The pseudo-time arrow model of phenomenal time describes this experience as a graph in which the local patterns of implicit causality form a cycle at the global scale. Thus the phenomenal past and future merge at their tails and one inhabits an experiential world that seems to be infinitely-repeating. Time Branching (QRI term; ref: 1, 2): A rare variant of exotic phenomenal time in which you feel like you are able to experience more than one outcome out of events that you witness. Your friend stands up to go to the bathroom. Midway there he wonders whether to go for a snack first, and you see “both possibilities play out at once in superposition”.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "In an extreme version of this experience type, each event seems to lead to dozens if not hundreds of possible outcomes at once, and your mind becomes like a choose-your-own-adventure book with a broccoli-like branching of narratives, and at the limit all things of all imaginable possible timelines seem to happen at once and you converge on a moment of eternity, thus transitioning out of this variety. We would like to note that a Qualia Computing article delved into the question of how to test if the effect actually allows you to see alternative branches of the multiverse. The author never considered this hypothesis plausible, but the relative ease of testing it made it an interesting, if wacky, research lead. The test consisted of trying to tell apart the difference between a classical and a quantum random number generator in real time. The results of the experiment are all null for the time being. World-Sheet (QRI term; ref: 1, 2): We represent modal and amodal information in our experience in a projective way. In most common cases, this information forms a 2D “sheet” that encodes the distance to the objects around you, which can be used as a depth-map to navigate your surroundings. A lot of the information we experience is in the combination of this sheet and phenomenal time (i.e. how it changes over time). Hyperbolic Phenomenal Space (QRI term; ref: 1, 2): The local curvature of the world-sheet encodes a lot of information about the scene.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "There is a sense in which the “energy” of the experience is related to the curvature of the world-sheet (in addition to its phenomenal richness and brightness). So when one raises the energy of the state dramatically (e.g. by taking DMT) the world-sheet tends to instantiate configurations with very high-curvature. The surface becomes generically hyperbolic, which profoundly alters the overall geometry of one’s experience. A lot of the accounts of “space expansion” on psychedelics can be described in terms of alterations to the geometry of the world-sheet. Dimensionality of Consciousness (QRI term; ref: 1, 2, 3): A generative definition for the dimensionality of a moment of experience can be “the highest virtual dimension implied by the network of correlations between globally bound degrees of freedom”. Admittedly, at the moment this is more of an intuition pump than a precise formalism, but a number of related phenomena suggest there is something in this general direction. For starters, differences between degrees of pain and pleasure are often described in terms of qualitative changes with phase transitions between them. Likewise, one generally experiences a higher degree of emotional involvement in a given stimuli the more sensory channels one is utilizing to interact with it.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Pleasure that has cognitive, emotional, and physical components in a coordinated fashion is felt as much more profound and significant than pleasure that only involves one of those “channels”, or even pleasure that involves all three but where they lack coherence between them. Another striking example involves the states of consciousness induced by DMT, in which there are phase-transitions between the levels. These phase transitions seem to involve a change in the dimensional character of the hallucinations: in addition to hyperbolic geometry, DMT geometry involves a wide range of phenomena with virtual dimensions. On lower doses the hallucinations take the shape of 2D symmetrical plane coverings. On higher doses those covers transform into 2.5D wobbly worldsheets, and on higher doses still into 3D symmetrical tessellations and rooms with 4D features. For example, the DMT level above 3D tessellations has its “walls” covered with symmetrical patterns that are correlated with one another in such a way that they generate a “virtual” 4th dimension, itself capable of containing semantic content. We suspect that one of the reasons why MDMA is so uniquely good at healing trauma is that in order to address a high-dimensional pain you need a high-dimensional pleasure to hold space for it. MDMA seems to induce a high-dimensional variety of feelings of wellbeing, which can support and smooth a high-dimensional pain like such as those which underly traumatic memories.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Qualia Futurology Meme (standard science/psychology term coined by Richard Dawkins; 1): A “meme” is a cultural unit of information capable of being transmitted from one mind to another. Examples of memes include jokes, hat styles, window-dressing color palettes, and superstitions. Memeplex (lesser known term coined by Richard Dawkins; 1, 2): A “memeplex” is a set of memes that, when simultaneously present, increase their ability to replicate (i.e. to be spread from one mind to another). Memeplexes do not need to say true things in order to be good at spreading; many strategies exist to motivate humans to share memes and memeplexes, ranging from producing good feelings (e.g. jokes), being threatening (e.g. apostasy), to being salient (e.g. famous people believe in them). A classic example of a memeplex is that of an ideology such as libertarianism, communism, capitalism, etc. Full-Stack Memeplex (QRI term; ref: 1, 2): A “full-stack memeplex” is a memeplex that provides an answer to most common human questions. While the scope of a memeplex like “libertarianism” extends across a variety of fields including economics and ethics, it is not a full-stack memeplex because it does not attempt to answer questions such as “why does anything exist? ”, “why are the constants of nature the way they are? ” and “what happens after we die? ”. Religions and some philosophies like existentialism, Buddhism, and the LessWrong Sequences are full-stack memeplexes.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "We also consider the QRI ecosystem to contain a full-stack memeplex.\n", + "\n", + "Hedonistic Imperative (coined by David Pearce; ref: 1, 2): The Hedonistic Imperative is a book-length internet manifesto written by David Pearce which outlines how suffering will be eliminated with biotechnology and why our biological descendants are likely to be animated by gradients of information-sensitive bliss.\n", + "Abolitionism (coined by David Pearce; ref: 1): In the context of transhumanism, Abolitionism refers to the view in ethics that we should eliminate all forms of involuntary suffering both in human and non-human animals alike. The term was coined by David Pearce.\n", "\n", "\n", "\n", "\n", "\n", "\n", + "Fast Euphoria (QRI term; ref: 1): This is one of the main dimensions along which a drug can have effects, roughly described as “high-energy and high-valence” (with high-loading terms including: energetic, charming, stimulating, sociable, erotic, etc.).\n", + "Slow Euphoria (QRI term; ref: 1): This is one of the main dimensions along which a drug can have effects, roughly described as “low-energy and high-valence” (with high-loading terms including: calming, relieving, blissful, loving, etc.).\n", + "Spiritual/Philosophical Euphoria (QRI term; ref: 1, 2): This is one of the main dimensions along which a drug can have effects, roughly described as “high-significance and high-valence” (with high-loading terms including: incredible, spiritual, mystical, life-changing, interesting, colorful, etc.).\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", "\n", - " - Title: Official Less Wrong Redesign: Nearly there, Author: matt, Date published: 2011-05-24, URL: https://www.lesswrong.com/posts/FXMQWrkNKcZNagZRS/official-less-wrong-redesign-nearly-there\n", + "Wireheading (standard psychology, neuroscience, and philosophy term; 1, 2): The act of modifying a mind’s reward architecture and hedonic baseline so that it is always generating experiences with a net positive valence (whether or not they are mixed). Wireheading Done Right (QRI term; ref: 1, 2): Wireheading done in such a way that one can remain rational, economically productive, and ethical. In particular, it entails (1) taking into account neurological negative feedback systems, (2) avoiding reinforcement cycles that narrow one’s behavioral focus, and (3) preventing becoming a pure replicator (see below). A simple proof of concept reward architecture for Wireheading Done Right is to cycle between different kinds of euphoria, each with immediate diminishing returns, and with the ability to make it easier to experience other kinds of euphoria. This would give rise to circadian cycles with stages involving fast, slow, and spiritual/philosophical euphoria at different times. Wireheading Done Right entails never getting stuck while always being in a positive state. Pure Replicator (QRI term; 1, 2): In the context of agents and minds, a Pure Replicator is an intelligence that is indifferent towards the valence of its conscious states and those of others. A Pure Replicator invests all of its energy and resources into surviving and reproducing, even at the cost of continuous suffering to themselves or others.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Its main evolutionary advantage is that it does not need to spend any resources making the world a better place. Consciousness vs. Replicators (QRI term; 1, 2): This is a reframe of the big-picture narrative of the meaning of life in which the ultimate battle is between the act of reproducing for the sake of reproduction and the act of seeking the wellbeing of sentient beings for the sake of conscious value itself. Maximum Effector (QRI term; 1): A Maximum Effector is an entity that uses all of its resources for the task of causing large effects, irrespective of what they may be. There is a sense in which most humans have a Maximum Effector side. Since causing large effects is not easy, one can reason that for evolutionary reasons people find such an ability to be a hard-to-fake signal of fitness. Arrogance and power may not be all that people find attractive, but they do play a role in what makes someone seem sexy to others. Hence why, unfortunately, people research how to cause large effects even if they are harmful to everyone. The idealized version of a Maximum Effector, however, would be exclusively interested in causing large effects to happen rather than doing so as a way to meet an emotional need among others.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Although being a Maximum Effector may seem crazy and pointless, they are important to consider in any analysis of the future because the long-tailed nature of large effects suggest that those who specifically seek to cause them are likely to have an impact on reality orders of magnitude higher than the impact of agents who try to simultaneously have both large and good effects.\n", + "\n", + "Sasha Shulgin Super-Shulgin Academy (coined by David Pearce; ref: 1, 2, 3, 4, 5, 6, 7, 8): This is a hypothetical future intellectual society that investigates consciousness empirically.\n", + "\n", + "Rather than merely theorizing about it or having people from the general population describe their odd experiences, the Super-Shulgin Academy directly studies the state-space of consciousness by putting the brightest minds on the task.\n", + "\n", + "The Super-Shulgin Academy (1) trains high-quality consciousness researchers and psychonauts, (2) investigates the computational trade-offs between different states of consciousness, (3) finds new socially-useful applications for exotic states of consciousness, (4) practices the art and craft of creating ultra-blissful experiences, and (5) develops and maintains a full-stack memeplex that incorporates the latest insights about the state-space of consciousness into the most up-to-date Theory of Everything.\n", + " - Title: Glossary of Qualia Research Institute Terms, Author: algekalipso, Date published: Thu, 22 Aug 2019, URL: https://qualiacomputing.com/2019/08/21/glossary-of-qualia-research-institute-terms/\n", + "\n", + "Link post Contents - An exemplar crisis with a timescale of months - Crisis and opportunity - Default example for humanity thinking about large-scale risk - Conclusion *Crossposted from EA forum. The second post in the **sequence** covers the importance of crises, argues for crises as opportunities, and makes the claim that this community is currently better at acting with longer timescale OODA loops but lacks skills and capabilities to act with short OODA loops. *We often talk about the hinge of history, a period of high influence over the whole future trajectory of life. If we grant that our century is such a hinge, it’s unlikely that the \"hinginess\" is distributed uniformly across the century; instead, it seems much more likely it will be concentrated to some particular decades, years, and months, which will have much larger influence. It also seems likely that some of these \"hingy\" periods will *look* eventful and be understood as crises at the time. So understanding crises, and the ability to act during crises, may be particularly important for influencing the long-term future. The first post in this sequence mentioned my main reason to work on COVID: it let me test my models of the world, and so informed my longtermist work. This post presents some other reasons, related to the above argument about hinges.\n", + " - Title: Hinges and crises, Author: Jan_Kulveit, Date published: 2022-03-29, URL: https://www.lesswrong.com/posts/XrGwrC9n8sDgXimcJ/hinges-and-crises\n", + "\n", + "None of these reasons would have been sufficient for me personally on their own, but they still carry weight, and should be sufficient for others in the next crisis. [1] ## An exemplar crisis with a timescale of months COVID has commonalities with some existential risk scenarios. (See Krakovna. - the crisis unfolds over a similar timescale (weeks or years, rather than seconds or hours), - governments have some role, - the risk is at least partially visible, - the general population is engaged in some way. This makes COVID a more useful comparison for versions of continuous AI takeoff where governments are struggling to understand an unfolding situation, but in which they have options to act and/​or regulate. Similarly, it is a useful model for versions of any x-risk where a large fraction of academia suddenly focuses on a topic previously studied by a small group, and resources spent on the topic increase by many orders of magnitude. This emergency research push is likely in scenarios with a warning shot or sufficiently loud fire alarm that gets noticed by academia. On the other hand, lessons learned from COVID will be correspondingly less useful for cases where few of the above assumptions hold (e.g. \"an AI in a box bursts out in an intelligence explosion on the timescale of hours\"). ## Crisis and opportunity Crises often bring opportunities to change the established order, and, for example, policy options that were outside the Overton window can suddenly become real.\n", + " - Title: Hinges and crises, Author: Jan_Kulveit, Date published: 2022-03-29, URL: https://www.lesswrong.com/posts/XrGwrC9n8sDgXimcJ/hinges-and-crises\n", + "\n", + "(This was noted pre-COVID by Anders Sandberg. ) There can also be rapid developments in relevant disciplines and technologies. Some examples of Overton shifts during COVID include: total border closures (in the West), large-scale and prolonged stay-at-home orders, mask mandates, unconditional payouts to large fractions of the population, and automatic data-driven control policies. Technology developments include the familiar new vaccine platforms (mRNA, DNA) going to production, massive deployment of rapid tests, and the unprecedented use of digital contact tracing. (Note that many other opportunities which opened up were not acted on. ) Taking advantage of such opportunities may depend on factors such as \"do we have a relevant policy proposal in the drawer\"? , \"do we have a team of experts able to advise\"? or \"do we have a relevant network\"? . These can be prepared in advance. ## Default example for humanity thinking about large-scale risk COVID will likely become the go-to example of a large-scale, seemingly low-probability risk we were unprepared for. The ability to shape narratives and attention around COVID could be important for the broader problem of how humanity should deal with other such risks.\n", + " - Title: Hinges and crises, Author: Jan_Kulveit, Date published: 2022-03-29, URL: https://www.lesswrong.com/posts/XrGwrC9n8sDgXimcJ/hinges-and-crises\n", + "\n", + "While there is a clear philosophical distinction between existential risks and merely catastrophic risks, 1) in practice it may be difficult to tell the ultimate scale of some risks, and 2) most people will not understand the distinction between GCRs and x-risks in an intuitive way (understanding both as merely \"extremely large\"). So narratives and research surrounding GCRs are important for work on x-risk. ## Conclusion The above are why it made sense to pay attention to COVID, even if the pandemic’s direct impact on the trajectory of humanity is small. (In some ways it still makes sense to pay attention. ) The broader conclusion is that longtermists’ ability to observe, orient themselves, decide and act during crises may be critical to influencing long-term outcomes. The usual ontology of longtermist interventions partitions the space according to \"cause areas\" or \"risks\", leaving room for the unknown \"cause X\". An alternative, almost orthogonal view partitions interventions according to the time scale of the OODA loop (i.e. the decision and action process) they implement. On this view, longtermism has so far focussed on actions in the top row, that have OODA loops on the horizon of years and decades. Typical examples might be writing books that fix the basic framing of a field, basic research, or community building.\n", + " - Title: Hinges and crises, Author: Jan_Kulveit, Date published: 2022-03-29, URL: https://www.lesswrong.com/posts/XrGwrC9n8sDgXimcJ/hinges-and-crises\n", + "\n", + "While there is a lot of commonality in actions along a column (e.g. at all timescales, the AI risk field will want to do AI research), there is also a lot that would be common interventions across a *row *(e.g. all cause areas may will need to know how governement may pass emergency regulation on a timescale of days).\n", + "\n", + "The skills and capabilities needed to act on a scale of months, weeks, or days seem relatively undeveloped.\n", + "\n", + "The following posts will make specific suggestions for what to improve in this regard, based on our experience with COVID—in particular the rather obvious suggestion of creating a longtermist *\"emergency response team\"* devoted to fast action.\n", + "\n", + "At the same time, I suggest taking this framing as a prompt: what else are we not doing?\n", + "\n", + "Where else is the table filled less than it should be?\n", + "\n", + " - ^I worked on the covid crisis at the expense of working directly on AI alignment and macro strategy at FHI, which is a very high bar.\n", + "\n", + "\n", + " - Title: Hinges and crises, Author: Jan_Kulveit, Date published: 2022-03-29, URL: https://www.lesswrong.com/posts/XrGwrC9n8sDgXimcJ/hinges-and-crises\n", "\n", "Link post\n", "\n", - " - Title: Instrumental Rationality and Overriding Defaults, Author: lifelonglearner, Date published: 2017-01-19, URL: https://www.lesswrong.com/posts/AT4NjNxav9TGoBSZu/instrumental-rationality-and-overriding-defaults\n", "\n", - "The word \"optimizer\" can be used in at least two different ways. First, a system can be an \"optimizer\" in the sense that it is solving a computational optimization problem. A computer running a linear program solver, a SAT-solver, or gradient descent, would be an example of a system that is an \"optimizer\" in this sense. That is, it runs an *optimization algorithm*. Let \"optimizer_1\" denote this concept. Second, a system can be an \"optimizer\" in the sense that it optimizes its environment. A human is an optimizer in this sense, because we robustly take actions that push our environment in a certain direction. A reinforcement learning agent can also be thought of as an optimizer in this sense, but confined to whatever environment it is run in. This is the sense in which \"optimizer\" is used in posts such as this. Let \"optimizer_2\" denote this concept. These two concepts are distinct. Say that you somehow hook up a linear program solver to a reinforcement learning environment. Unless you do the \"hooking up\" in a particularly creative way there is no reason to assume that the output of the linear program solver would push the environment in a particular direction. Hence a linear program solver is an optimizer_1, but not an optimizer_2. On the other hand, a simple tabular RL agent *would* eventually come to systematically push the environment in a particular direction, and is hence an optimizer_2.\n", - " - Title: Two senses of \"optimizer\", Author: Joar Skalse, Date published: 2019-08-21, URL: https://www.lesswrong.com/posts/rvxcSc6wdcCfaX6GZ/two-senses-of-optimizer\n", + "Last week I wrote \n", + "a short\n", + "note encouraging people to revisit disaster plans in light of \n", "\n", - "However, such a system does not run any internal optimization algorithm, and is therefore not an optimizer_1. This means that a system can be an optimizer_1 while not being an optimizer_2, and vice versa. There are some arguments related to AI safety that seem to conflate these two concepts. In *Superintelligence *(pg 153), on the topic of Tool AI, Nick Bostrom writes that: > A second place where trouble could arise is in the course of the software’s operation. If the methods that the software uses to search for a solution are sufficiently sophisticated, they may include provisions for managing the search process itself in an intelligent manner. In this case, the machine running the software may begin to seem less like a mere tool and more like an agent. Thus, the software may start by developing a plan for how to go about its search for a solution. The plan may specify which areas to explore first and with what methods, what data to gather, and how to make best use of available computational resources. In searching for a plan that satisfies the software’s internal criterion (such as yielding a sufficiently high probability of finding a solution satisfying the user-specified criterion within the allotted time), the software may stumble on an unorthodox idea. For instance, it might generate a plan that begins with the acquisition of additional computational resources and the elimination of potential interrupters (such as human beings).\n", - " - Title: Two senses of \"optimizer\", Author: Joar Skalse, Date published: 2019-08-21, URL: https://www.lesswrong.com/posts/rvxcSc6wdcCfaX6GZ/two-senses-of-optimizer\n", - "\n", - "To me, this argument seems to make an unexplained jump from optimizer_1 to optimizer_2. It begins with the observation that a powerful Tool AI would be likely to optimize its internal computation in various ways, and that this optimization process could be quite powerful. In other words, a powerful Tool AI would be a strong optimizer_1. It then concludes that the system might start pursuing convergent instrumental goals – in other words, that it would be an optimizer_2. The jump between the two is not explained. The implicit assumption seems to be that an optimizer_1 could turn into an optimizer_2 unexpectedly if it becomes sufficiently powerful. It is not at all clear to me that this is the case – I have not seen any good argument to support this, nor can I think of any myself. The fact that a system is internally running an optimization algorithm does not imply that the system is selecting its output in such a way that this output optimizes the environment of the system. The excerpt from *Superintelligence* is just one example of an argument that seems to slide between optimizer_1 and optimizer_2. For example, some parts of *Dreams of Friendliness *seem to be doing so, or at least it’s not always clear which of the two is being talked about. I’m sure there are more examples as well. Be mindful of this distinction when reasoning about AI. I propose that \"consequentialist\" (or perhaps \"goal-directed\") is used to mean what I have called \"optimizer_2\".\n", - " - Title: Two senses of \"optimizer\", Author: Joar Skalse, Date published: 2019-08-21, URL: https://www.lesswrong.com/posts/rvxcSc6wdcCfaX6GZ/two-senses-of-optimizer\n", - "\n", - "I don’t think there is a need for a special word to denote what I have called \"optimizer_1\" (at least not once the distinction between optimizer_1 and optimizer_2 has been pointed out). Note: It is possible to raise a sort of embedded agency-like objection against the distinction between optimizer_1 and optimizer_2. One might argue that: > There is no sharp boundary between the inside and the outside of a computer. An \"optimizer_1\" is just an optimizer whose optimization target is defined in terms of the state of the computer it is installed on, whereas an \"optimizer_2\" is an optimizer whose optimization target is defined in terms of something outside the computer. Hence there is no categorical difference between an optimizer_1 and an optimizer_2. I don’t think that this argument works. - A computer that is able to very quickly solve very large linear programs. - A computer that solves linear programs, and tries to prevent people from turning it off as it is doing so, etc. System 1 is an optimizer_1 that solves linear programs, whereas system 2 is an optimizer_2 that is optimizing the state of the computer that it is installed on. These two things are different. (Moreover, the difference isn’t just that system 2 is \"more powerful\" than system 1 – system 1 might even be a better linear program solver than system 2.\n", - " - Title: Two senses of \"optimizer\", Author: Joar Skalse, Date published: 2019-08-21, URL: https://www.lesswrong.com/posts/rvxcSc6wdcCfaX6GZ/two-senses-of-optimizer\n", - "\n", - ") Acknowledgements: We were aware of the difference between \"optimizer_1\" and \"optimizer_2″ while working on the mesa-optimization paper, and I’m not sure who first pointed it out.\n", - "\n", - "We were also probably not the first people to realise this.\n", - " - Title: Two senses of \"optimizer\", Author: Joar Skalse, Date published: 2019-08-21, URL: https://www.lesswrong.com/posts/rvxcSc6wdcCfaX6GZ/two-senses-of-optimizer\n", - "\n", - "In the first weekend of this year, the Future of Life institute hosted a landmark conference in Puerto Rico: \"The Future of AI: Opportunities and Challenges\". The conference was unusual in that it was not made public until it was over, and the discussions were under Chatham House rules. The slides from the conference are now available. The list of attenders includes a great many famous names as well as lots of names familiar to those of us on Less Wrong: Elon Musk, Sam Harris, Margaret Boden, Thomas Dietterich, all three DeepMind founders, and many more.\n", - "\n", - "This is shaping up to be another extraordinary year for AI risk concerns going mainstream!\n", - "\n", - " - Title: Slides online from \"The Future of AI: Opportunities and Challenges\", Author: Paul Crowley, Date published: 2015-01-16, URL: https://www.lesswrong.com/posts/TJSRiqBJxbcwEo7AR/slides-online-from-the-future-of-ai-opportunities-and\n", - "\n", - "Erik DeBenedictis on supercomputing \n", - " Erik DeBenedictis works for Sandia’s Advanced Device Technologies department. He has been a member of the International Technology Roadmap for Semiconductors since 2005.\n", - "DeBenedictis has received Ph.D. in computer science from Caltech. As a grad student and post-doc, he worked on the hardware that turned into the first hypercube multiprocessor computer. Later dubbed the “Cosmic Cube,” it ran for more than a decade after he left the university and was copied over and over. It’s considered the ancestor of most of today’s supercomputers.\n", - "In the 1980s, then working for Bell Labs in Holmdel, N.J., DeBenedictis was part of a consortium competing for the first Gordon Bell award. The team got the second place award, the first place going to Sandia. During the 1990s, he ran NetAlive, Inc., a company developing information management software for desktops and wireless systems. Starting in 2002, DeBenedictis was one of the project leads on the Red Storm supercomputer. \n", - "The opinions expressed by Erik below are his own and not those of Sandia or the US Department of Energy. This document has been released by Sandia as SAND Number 2014-2679P. Luke Muehlhauser: Some of your work involves reversible computing, which I previously discussed with Mike Frank. Mike’s view seemed to be that there were promising signs that reversible computing would be possible eventually, but progress is not moving quickly due to lack of funding and interested researchers.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "Is that your view as well? And based on my interview with him, do you seem to have a substantially different impression that Mike does about anything he and I discussed? Erik DeBenedictis: I agree with Mike, but his discussion of minimum energy in computing due to irreversibility is just part of a larger topic of minimum energy in computing that starts with “Moore’s Law Ending”. For any reader who has not read Mike Frank’s interview, I’d like to give a quick summary of the relevant points. Mike was interviewed about reversible logic, which is sometimes called reversible computing. If you were a brilliant engineer and could figure out how to make a computer logic gate like AND or OR that dissipated kT joules per logic operation (the meaning of kT is in the next paragraph), you would discover that there is an additional heat production on the order of kT due to the interaction between information and thermodynamics. If you were determined to make even lower power computer gates anyway, you would have to use reversible logic principles. You could use a different universal gate set that would include a new gate such as the TOFFOLI or FREDKIN gate. You could also use regular gates (e. g. AND, OR, NOT) and a “retractile cascade” clocking scheme that reverses the computation after you capture the answer. For reference on kT: k = 1.38 x 10-23 Joules/Kelvin is Boltzmann’s constant and T is the absolute temperature with T = 300 Kelvin at room temperature.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "kT is about 4 zeptojoules = 4 x 10-21 Joules. Comparing this number to today’s computers is imprecise because dissipation in today’s computers is primarily attributable to the interconnect wire, which varies in length. An AND or OR gate in a modern computer may dissipate a million times this value. A great many respected scientists believe that reversible computing is feasible, but challenging. If their views are correct, computation should be possible at “arbitrarily low energy levels” and all theories proposing unavoidable, general limits are incorrect. There are a handful of contrary theories proposing minimum energy dissipation levels for computation. Several key ones are Landauer’s Limit of “on the order of kT” per logic operation1, a thermal limit of 40-100 kT (depending on your definition of reliable), and the concept in the popular press today that “Moore’s Law is Ending” and the minimum energy per computation is whatever is in the rightmost column of the International Technology Roadmap for Semiconductors (ITRS). That value is about 50,000 kT with typical lengths of interconnect wire. Scientific situations with multiple competing theories can be settled by a scientific experiment. For example, there is a researcher in New York that has a superconducting circuit running in the sub-kT range and looks like it could demonstrate a logic circuit in another couple “spins” of his chip.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "Demonstrating and rigorously measuring a sub-kT circuit would invalidate all current theories claiming unavoidable limits. Whether anybody will fund such an experiment should depend on whether anybody cares about the result, and I’d like to present two society-level questions that the experiment would resolve: The computer industry started its upward trend during WWII, growing industry revenue and computer throughput in a fairly clean exponential lasting 70 years. The revenue from semiconductors and downstream industries is around $7 trillion per year right now. If there is a lower energy limit to computing, the shift in growth rate will cause a glitch in the world’s economy. My argument is that proving or disproving theories of computing limits could be accomplished for a very small fraction of $7 trillion per year. The second has to do with profoundly important computational problems, such as the simulation of the global environment to assess climate change issues. Existing climate models running on petaflops supercomputers give varying projections for the future climate, with these projections diverging from observations over the last decade. Regardless of politics, the remedy would be a more sophisticated climate model running on a bigger supercomputer. We don’t know how much bigger, but a zettaflops or more has been mentioned in this context.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "If any of the minimum energy dissipation theories are correct, the energy dissipation of the required supercomputer could turn out to be too large and climate modeling may be infeasible; if the theory that computing is possible at “arbitrarily low levels” is true, accurate climate modeling will just require a highly-advanced computer.\n", - "\n", - "I’ve tried to expand on Mike’s point: Research on reversible computing could shed light on the future of the economy and the planet’s climate, but I do not know of a single person funded for reversible computing research.\n", - "\n", - "Furthermore, a conclusive demonstration of reversible computing would show that there is plenty of room for improving computer efficiency and hence performance.\n", - "\n", - "If “Moore’s Law is Ending” means an end to improving computer efficiency, validating reversible computing would show this to be a matter of choice not technology.\n", - "\n", - "Luke: From your perspective, what are the major currently-foreseeable barriers that Moore’s law might crash into before hitting the Landauer limit? (Here, I’m thinking more about the economically important “computations per dollar” formulations of Moore’s law rather than the “serial speed” formulation, which hit a wall in 2004.)\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "Erik: There is huge upside, but not necessarily for every application. The “computations per dollar” link in the question focused on the use of computers as a platform for strong Artificial Intelligence (AI), so I will comment specifically on that application: I wouldn’t be surprised to see AI accelerated by technology specifically for learning, like neural networks with specialized devices for the equivalent of synapses.\n", - "Let’s consider (a) Moore’s Law 1965 to say 2020 and (b) Beyond Moore’s Law 2020+.\n", - "From 1965 to 2020, the strategy was to shrink line width. That strategy will be good for 1012 or so increase in computations per dollar.\n", - "I see the following classes of advances beyond 2020 that will each give maybe 10-100x efficiency increase each: More efficient implementation of the von Neumann architecture. More parallelism, with a commensurate increase in the difficulty of programming. Software improvements for more efficient execution (e. g. new computer languages and compilers to run general code on Graphics Processing Units). Better algorithms that solve a given problem with fewer computations. Accelerators, such as CPU+GPU today, extendable to CPU+GPU+various new accelerator types. Even at constant energy per gate operation, continued scaling in 2D and better use of the third dimension for reducing communications energy. Optical interconnect has upside, but optics is often oversold.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "Nanodevices with behavior different from a transistor that allow some computer functions to be done more efficiently. Examples: Memristors, analog components. Improved gate technology through adiabatic methods, sub-threshold or low-threshold operation, or probabilistic computing. Eventually, reversible computing (note below on this one). Alternative models of computation (i.e. neuromorphic) that do not use gates as normally defined. If the ten items in the list above yield an average of 1½ orders of magnitude increase in computations per dollar each, you have more upside than the entire run of Moore’s Law. If a couple of the items in the list don’t pan out, you could achieve excellent results by concentrating on other paths. So I do not see a general technology crash anytime soon. However, certain specific applications may be dependent on a just a subset of the list above (climate modeling was mentioned) and could be vulnerable to a limit. Reversible computing plus continued reduction in manufacturing cost per device could extend upside potential tremendously. However, the necessary technology investment will be greater in the future for a less clear purpose. The message of Moore’s Law was very concise: industry and government invest in line width shrinkage and get a large payoff. In the future, many technology investments will be needed whose purposes have less clear messages.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "Bottom line: In general, the path ahead is expensive but will yield a large increase in computations per dollar. Specific application classes could see limits, but they will have to be analyzed specifically. Luke: What’s your opinion on whether, in the next 15 years, the dark silicon problem will threaten Moore’s Law (computations per dollar)? Erik: I believe the dark silicon problem will negatively impact computations per dollar. The problem and the underlying energy efficiency problem are going to get worse at least until the cost of increased energy is greater than the cost of refining a solution and bringing it to production. That will happen eventually, but I believe the problem will persist longer than may be expected due momentum against change. However, you admit Moore’s Law has ended when you admit that there is a dark silicon problem. The underlying cause of dark silicon is that technology scales device dimensions faster than it reduces power. This causes power per unit chip area to increase, which contradicts the key statement in Gordon Moore’s 1965 paper that defined Moore’s Law: “In fact, shrinking dimensions on an integrated structure makes it possible to operate the structure at higher speed for the same power per unit area”. The mismatched scaling rates create a problem for computations per dollar. Today, the cost of buying a computer is approximately equal to the cost of supplying it with power over its lifetime.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "Unless power efficiency can be increased, improvements to computer logic will not benefit the user because the amount of computation they use will be limited by the power bill. The mismatched scaling rates can be accommodated (but not solved) by turning off transistors (dark silicon), packing microprocessors with low energy-density functions like memory (a good idea, to a point), and specialization (described in your interview under dark silicon problem). The scaling rates could be brought together by more power-efficient transistors, such as the Tunnel Field Effect Transistor (TFET). However, this transistor type will only last a few generations. See here. Theory says energy per computation can be made “arbitrarily small,” but R&D to exploit these issues will be expensive and disruptive. The leading approaches I am aware of are: Adiabatic. A fundamentally different approach to logic gate circuits. Example: Mike Frank’s 2LAL. Certain low-voltage logic classes: For example, see CMOS LP in arXiv 1302.0244 (which is not same as ITRS CMOS LP). Reversible computing, the topic of Mike Frank’s interview. The approaches above are disruptive, which I believe limits their popularity today. The approaches use different circuits from CMOS, which would require new design tools. New design tools would be costly to develop and would require retraining of the engineers that use them.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "Children learn the words “and” and “or” when they are about one year old, with these words becoming the basis of AND and OR in the universal logic basis of computers. To exploit some technologies that save computer power, you have to think in terms of a different logic basis like TOFFOLI, CNOT, and NOT. Some of the ideas above would require people to give up concepts that they learned as infants and have not had reason to question before. Luke: What do you mean by “you admit Moore’s Law has ended when you admit that there is a dark silicon problem”? The computations-per-dollar Moore’s Law has held up at least through early 2011 (I haven’t checked the data after that), but we’ve known about the dark silicon problem since 2010 or earlier. Erik: Moore’s Law has had multiple meanings over time, and is also part of a larger activity. There was a very interesting study by Nordhaus that revealed the peak computation speed of large computers experienced an inflection point around WW II and has been on an upwards exponential ever since. Eyeballing figure 2 of his paper, I’d say the exponential trend started in 1935. Gordon Moore published a paper in 1965 with the title “Cramming more Components onto Integrated Circuits” that includes a graph of components per chip versus year.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "As I mentioned for a previous question, the text of the paper includes the sentence, “In fact, shrinking dimensions on an integrated structure makes it possible to operate the structure at higher speed for the same power per unit area”. The graph and previous sentence seem to me to be a subjective description of an underlying scaling rule that was formalized by Dennard in 1974 and is called Dennard scaling. I have sketched below Moore’s graph of components as a function of year with Nordhaus’ speed as a function of year on the same axes (a reader should be able to obtain the original documents from the links above, which are more compelling than my sketch). This exercise reveals two things: (1) the one-year doubling period in Moore’s paper was too fast, and is now known to be about 18 months, and (2) that Moore’s Law is a subtrend of the growth in computers documented by Nordhaus. A really interesting question is whether Moore was applying somebody else’s law or whether the two laws were actually part of a larger concept that was not understood at the time. I conclude the latter. Intel did not invent the microprocessor until six years after Moore’s article. I have also talked to people (not Moore) who tell me Gordon Moore was thinking about general electrical circuits and was not foreseeing the emergence of the microprocessor. Let me try to apply Moore’s Law as defined by his paper.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "I recall building a computer system in 1981 with an 8086 (very similar to the 8088 in the original IBM PC). I’d heard it was highly complex and dissipated a lot of heat, so I put my finger on it to experience the heat. I recall surprise that it didn’t seem warmer than anything else. I have thought about the heat from microprocessors in the last year, 33 years later. Since Moore’s Law says power per unit area is the same and chips are nearly the same size at 1 cm2, I should be able to put my finger on a chip and not feel any heat. The reality is that there is a new structure sitting on top of today’s microprocessors that reminds me of Darth Vader’s head and is called a “heat sink”. The heat sink is to remove 50-200 watts of heat generated by the chip. I believe I’ve just made a case that any microprocessor with a heat sink violates Moore’s Law. What’s going on? Moore’s Law is being given additional meaning over and above what Moore was thinking. Many people believe Moore’s Law is only about dimensional scaling, a conclusion supported by the title of his article and the main graph. Moore’s Law has also been associated with computations per dollar, but that law had been around for 30 years before Moore’s paper. I found the interview with Hadi Esmaeilzadeh on Dark Silicon to be on track, yet he uses another interpretation of Moore’s Law – one where Moore’s Law continues, but Dennard scaling ended in the mid-2000s.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "Yet, I quoted the phrase from Moore’s paper that disclosed the scaling rule that later became known as Dennard scaling. At a higher level, I believe Moore’s Law has turned into a marketing phrase that is being redefined as needed by the semiconductor industry so it remains true. So why are computations per dollar rising? For many years, the vendor objective was to make processors that ran word processors and web browsers faster. This trend culminated in the early 2000s with processors like the Pentium IV with a 4 GHz clock and dissipating 200W+. Customers rebelled and industry shifted to multicore. With an n-core microprocessor, the results of running the benchmark on one core could be multiplied by n. This is an example of progress (raising computations per dollar) by item 2 in my response to a previous question (more parallelism, subject to difficulty in programming). Even now, most software does not exploit the multiple cores. Luke: You write that “Customers rebelled and industry shifted to multicore”. I typically hear a different story about the 2002-2006 era, one that didn’t have much to do with customer rebellion, but instead the realization by industry that the quickest way to keep up the Moorean trend — to which consumers and manufacturers had become accustomed — was to jump to multicore. That’s the story I see in e.g. The Future of Computing Performance by National Academies Press (official summary here).\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "Moreover, the power scaling challenge to Moore’s Law was anticipated many years in advance by the industry, for example in the ITRS reports.\n", - "\n", - "Can you clarify what you mean by “customers rebelled”?\n", - "\n", - "Erik: What happens if you take projections of the future to be true and then the projections change? You eventually end up with multiple “truths” about the same thing in the historical record. I accept that the stories you hear are true, but there is another truth based on different projections.\n", - "Let us mathematically invert the ITRS roadmap to see how projections of today’s (2014) microprocessor clock rate evolved as industry addressed power scaling and shifted to multicore. I have gone back to earlier editions of the ITRS and accessed edition reports for 2003, 2005, and 2007. In table 4 of the executive summary of each edition, they have a projection of “on chip local clock,” which means microprocessor clock rate. I accessed Pricewatch to get the 2014 clock rate.\n", + "Russia’s\n", + "invasion of Ukraine. I wanted to follow up to share a bit about\n", + "what this has meant for us:\n", "\n", "\n", "\n", "\n", "\n", + "\n", + " - I read some about how targets might be chosen and looked at\n", + "fallout maps. I wasn’t able to find anything especially high\n", + "quality. In a \"destroy military targets\" scenario Boston-area things\n", + "are probably pretty far down the list; in a \"destroy major cities\"\n", + "scenario less so. Nukemap\n", + "can be useful for looking at how likely a bomb hitting various\n", + "locations is to kill you in the initial blast, which affects\n", + "trade-offs around getting out vs preparing to potentially shelter at\n", + "home. My reading is that our house in particular is close enough to\n", + "Boston that we would be unlikely to survive a bomb targeted there. If\n", + "you are at an intermediate distance from a target, where are you are\n", + "unlikely to be killed in the initial blast but likely to receive\n", + "fallout, it’s worth planning for the ~15min window\n", + "between the blast and fallout arriving.\n", + "\n", + "\n", + "\n", + " - We talked as a house about\n", + "how we might handle various scenarios. This was especially necessary\n", + "since we share a car, though in most\n", + "disasters we would probably stay put. This also included\n", + "discussing what sort of escalation in Ukraine might make us leave\n", + "Boston, but we really didn’t get anywhere with that.\n", + " - Title: War-prompted Disaster Planning, Author: jefftk, Date published: 2022-03-13, URL: https://www.lesswrong.com/posts/Fsg9ZhCNRyaFq9rhb/war-prompted-disaster-planning\n", + "\n", + "\n", + "\n", + " - I had put together two emergency\n", + "backpacks in Fall 2017 when I was thinking about disasters, and\n", + "this seems like a good time to think about updating them. Some things\n", + "were close to expiration (emergency food, medication) and our needs\n", + "are somewhat different (8mo baby). Additionally, I hadn’t done a\n", + "great job choosing what to put in them the first time: I found this\n", + "very stressful and needed to just put some things together and be done\n", + "with it or else I wasn’t going to finish at all. Julia volunteered to handle this, for\n", + "which I’m very thankful. We also looked over our emergency supplies\n", + "in general and got some things we decided we were missing.\n", + "\n", + "\n", + "\n", + " - We’ve started a lists for additional things we might bring if evacuating by car.\n", + "\n", + "\n", + "\n", + " - I made paper instructions for how to use the solar system and battery/​inverter.\n", + "\n", + "\n", + "\n", + " - I bought water\n", + "containers for my dad to keep it his basement (we already have\n", + "some).\n", + "\n", + "\n", + "\n", + " - We moved some things around to make them more accessible in an\n", + "emergency. For example, things that we might want to bring with us\n", + "if either evacuating by car or sheltering in the basement (tent,\n", + "portable crib, air mattresses) are now stored in a convenient place,\n", + "when previously they were more scattered around the house.\n", + " - Title: War-prompted Disaster Planning, Author: jefftk, Date published: 2022-03-13, URL: https://www.lesswrong.com/posts/Fsg9ZhCNRyaFq9rhb/war-prompted-disaster-planning\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "A lot of these are not especially specific to the current situation,\n", + "but instead more are taking the time to think through what might be\n", + "useful in a range of disasters.\n", + "\n", + " \n", + "\n", + " - Title: War-prompted Disaster Planning, Author: jefftk, Date published: 2022-03-13, URL: https://www.lesswrong.com/posts/Fsg9ZhCNRyaFq9rhb/war-prompted-disaster-planning\n", + "\n", + "Contents\n", + " - Background\n", + " - Bounty\n", + " - What does count as a referral? \n", + " - How can I claim the bounty?\n", + " - Questions?\n", + "*Crossposted to* *EA Forum**TL;DR: 500 USD bounty for successful applicants referrals * \n", + "# Background\n", + "The **PIBBSS Summer Research Fellowship** is seeking applicants studying complex and intelligent behavior in natural and social systems, such as ecology, evolutionary biology, cognitive science, neuroscience, sociology, legal theory, political economy, statistical mechanics, social anthropology, linguistics, media studies, and more. \n", + "Accepted fellows will conduct AI alignment research under the mentorship of experienced alignment researchers. \n", + "You can learn more at https://​​www.pibbss.ai/​​ or by reading the post Introducing the Principles of Intelligent Behaviour in Biological and Social Systems (PIBBSS) Fellowship.\n", + "**The extended application deadline is Jan 23rd.**\n", + "# Bounty - We are offering a bounty of** 500 USD** for each applicant you refer who we end up accepting to the program. - We are offering a bounty of **100 USD** for each applicant you refer, and we end up interviewing (i.e. they get to stage 3 of the application process). We expect to accept 5-15 fellows and to interview 20-40 applicants. ## What does count as a referral? Roughly, it means that you were either the first one who informed someone about the fellowship or that your encouragement was the main reason they applied. ## How can I claim the bounty?\n", + " - Title: PIBBSS Fellowship: Bounty for Referrals & Deadline Extension, Author: Anna Gajdova, Date published: 2022-01-17, URL: https://www.lesswrong.com/posts/8uxbDaWBuygSfiuES/pibbss-fellowship-bounty-for-referrals-and-deadline\n", + "\n", + "Fill in this form and list the people you’ve referred for the fellowship, you can add details of your interactions.\n", + "\n", + "You should also encourage the people you referred to mention you in their application (we ask them how they learned about the fellowship).\n", + "\n", + "We’ll also try reaching out to people mentioned in applications, even when they don’t claim the bounty.\n", + "\n", + "In case the people you referred succeed, we’ll get in touch with you after the end of the application process in mid-March.\n", + "\n", + "The final decision about bounty eligibility is ours.\n", + "\n", + "Bounty can also be claimed retroactively, i.e. when you’ve already referred someone, and they applied.\n", + "\n", + "# Questions?\n", + "\n", + "If you have any questions about the bounty or the fellowship, you can ask in the comments or reach out to anna@pibbss.\n", + "\n", + "ai or contact@pibbss.\n", + "\n", + "ai.\n", + " - Title: PIBBSS Fellowship: Bounty for Referrals & Deadline Extension, Author: Anna Gajdova, Date published: 2022-01-17, URL: https://www.lesswrong.com/posts/8uxbDaWBuygSfiuES/pibbss-fellowship-bounty-for-referrals-and-deadline\n", + "\n", + "Contents\n", + " - America’s Interests\n", + " - China’s Interests\n", + " - Relative Power\n", + " - Conflict Points\n", + " - Ships vs Guided Missiles\n", + " - Taiwan\n", + " - Cyberspace\n", + " - Space War\n", + " - Protracted Total War is Unlikely\n", + "The Cold War is over. Russia is a fading power. The most important geopolitical rivalry of the 21st century is between China and the USA. Any analysis of the conflict must take into account the possibility that it escalates into a hot war. This post explores how a direct conflict between the USA and China might unfold. It assumes strong AI has not been invented and nuclear weapons are not used.\n", + "\n", + "# America’s Interests\n", + "\n", + "The United States’ interests have been basically unchanged since 1945. Its primary objective is to maintain the liberal world order (LWO), also known as the \"rules-based international order\". The LWO describes a set of global, rule-based structured relationships based on economic liberalism as embodied by the United Nations and the World Trade Organization. The LWO promotes political liberalism too, albeit much less consistently.\n", + "\n", + "As the primary power behind the LWO, the United States designed it to maximize economic and political power of the United States. As the United States’ relative power wanes, we may see a transition toward a more multipolar LWO.\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "# China’s Interests\n", + "\n", + "China’s interests have been basically unchanged since 1978. Its primary objective is to maintain internal domestic stability *i.e.* prevent regime change. There are two ways of keeping its population under control: via a police state and via economic development. The stronger it’s police state the less economic development is necessary and *vice versa*. China’s economic growth is slowing as its east cost gets closer to a Western standard of living.\n", + "\n", + "The People’s Republic of China did not get a seat at the table in 1945 when the LWO was designed. It wasn’t even allowed into the United Nations until 1971. From 1945 until 1971, \"China\" was represented by the Republic of China *i.e.* Taiwan. This illustrates how the LWO favors American geopolitical interests and is one of the many reasons why the People’s Republic of China seeks to annex Taiwan.\n", + "\n", + "China has prospered under the LWO. Rather than establishing broad international coalitions, China tends to pursue its interests bilaterally. With a few exceptions (like the dispute over the South China Sea) China is content to play according to the rules of the LWO.\n", + "\n", + "Besides annexing Taiwan, China’s geopolitical interests mostly revolve around securing markets and raw materials to fuel its economic development. This is the motivation behind 一带一路 (the Belt and Road Initiative).\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "# Relative Power\n", + "\n", + "The United States dominates global military spending.\n", + "\n", + "\n", + "\n", + "Military spending is a trailing indicator. What really matters is the size of each nation’s respective economy. Metaculus predicts a 1.7 China-to-USA GDP ratio in 2050. The PPP difference will be even higher.\n", + "\n", + "\n", + "\n", + "This underestimates the relative strength of the USA because the USA has many close allies: The European Union, Britain, Japan, South Korea, Taiwan and so on.\n", + "\n", + "\n", + "\n", + "China has North Korea.\n", + "\n", + "\n", + "\n", + "The advantage to America is it has allies to call upon. The disadvantage to America is it has allies it must coordinate with and defend. America is spread thin. China focuses its attention on its smaller sphere of influence.\n", + "\n", + "# Conflict Points\n", + "\n", + "Of all the potential points of conflict, the obvious ones are Taiwan and the South China Sea. For the purpose of this analysis, suppose Taiwan declares independence from China in 2050 which causes China to attack Taiwan which causes the United States to attack China.\n", + "\n", + "The last time something like this happened was in 1949 when the Kuomintang fled to Taiwan. The United States refused to get involved until the Korean War in 1950. The United States sent its Seventh Fleet into the Taiwan Straight. American naval supremacy kept the People’s Republic of China from advancing into Taiwan, thus establishing the status quo which remains until today.\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "The United States maintains naval dominance over China. This will not last. China’s PPP surpassed the United States years ago. China’s GDP will surpass the United States well before 2050. Most importantly, the crown jewels of the United States’ fleet will be almost useless in a direct conflict against China.\n", + "\n", + "# Ships vs Guided Missiles\n", + "\n", + "When the United States wants to project power to a faraway land it uses aircraft carriers. In the 1996 Taiwan Strait Crisis, the United States sent two aircraft carrier battle groups into the Taiwan Strait to demonstrate solidarity with Taiwan.\n", + "\n", + "Aircraft carriers dominated the seas in the second half of the 20th century because planes used to have short ranges and missiles used to be dumb. Today, airplanes have long ranges and missiles are smart.\n", + "\n", + "A Chinese DF-21D anti-ship ballistic missile is believed to have a range in excess of 1,500 km. The newer CH-AS-X-13 has a range of 3,000 km. It can be launched from an airplane so its effective range is even farther. I estimate anti-ship missiles cost about $2 million each. An aircraft carrier costs more than $10 billion. Hiding an aircraft carrier battle group on the open sea isn’t possible. The only way aircraft carriers could be remotely viable in a hot war between the United States and China would be if they had very reliable missile defense systems.\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "Israel’s Iron Dome blocked 90% to 95% of incoming missiles in this year’s Israel-Palestine crisis. That means it missed 5% to 10% of incoming missiles. A Hamas Qassam rocket is primitive. It is propelled by a mixture of sugar and potassium nitrate (ordinary fertilizer). It is not even accurate enough to use against military targets. A Qassam rocket is to a Chinese CH-AS-X-13 what a Mitsubishi A6M Zero is to an F-16.\n", + "\n", + "I predict at least 5% of guided missiles can penetrate a surface fleet’s missile defense system in 2050. If my numbers are right then either side can sink a $10 billion aircraft carrier with no more than $40 million in guided missiles.\n", + "\n", + "An Arleigh Burke-class destroyer costs $2 billion per ship. Surface ships are basically going to be obsolete in a direct war between superpowers. The battles of the sea will be fought with aircraft, missiles, drones and maybe submarines.\n", + "\n", + "# Taiwan\n", + "\n", + "I expect China will not be able to establish air supremacy far beyond its borders and that the United States will not be able to establish air supremacy over mainland China. Without air supremacy, neither side can protect aircraft carriers and transport fleets. This makes amphibious invasions difficult. Except Taiwan *isn’t* far beyond China’s borders.\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "Taiwan’s military can be quickly destroyed by a Chinese attack. Taiwan itself is too weak to repulse China on the beaches. If Taiwan wants to maintain its independence then it needs dug-in defenders. Taiwan’s mountainous terrain is advantageous here. Taiwan already conscripts all qualified males. It should be training them in guerrilla warfare. This is not happening. Taiwan should also have weapons caches and underground bunkers strewn all over the island. It doesn’t.\n", + "\n", + "How many troops would it take to conquer Taiwan?\n", + "\n", + "\n", + " - A Chinese invasion of Taiwan might be similar to a United States invasion of the Japanese homeland in 1945. In 1945, the United States’ Project Downfall projected an invasion force of 5 million was required to subdue a population of 70 million. Projecting from this, an invasion force of 2 million might to be necessary to subdue Taiwan’s population of 23 million.\n", + "\n", + " - On the other hand, armies today are more efficient than they were in 1945. (People are more docile too.) Afghanistan has a population of 40 million but US troop levels in Afghanistan reached a height of only 100 thousand. It might take China only 50 thousand troops to subdue Taiwan.\n", + "\n", + "I’m not sure which number to use. 2021 Taiwan is a lot like 2021 Japan but 2021 Japan is very unlike 1945 Japan. Taiwan is also unlike Afghanistan. It might take even fewer than 50 thousand troops to subdue Taiwan.\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "If 50 thousand troops is all it takes then China could land them quickly before the USA gets its act together. But if 2 million is what it takes to conquer Taiwan then China would need a bigger army than it has right now. Also, you can’t land an invasion force of 2 million troops overnight. China would have to win the initial missile exchange and then maintain air superiority while it landed troops in Taiwan.\n", + "\n", + "Things get hard to predict from here. What happens depends on the decisions of individual world leaders and the determination of various peoples.\n", + "\n", + "# Cyberspace\n", + "\n", + "In *Darknet Diaries Episode 21: Black Duck Eggs*, Ira Winkler tells the story of how his team broke into a major datacenter containing billions of dollars worth of technology.\n", + "\n", + "\n", + "> Well, the first time you steal a billion dollars it’s a bit of a rush. After you’ve done this so many times it’s almost expected. Frankly, it was really unclimactic to actually take over control of all their computers in the RND center.\n", + "\n", + "\n", + "I know, right? Who cares if you can just walk into a datacenter and steal a billion dollars worth of advanced technology? It doesn’t mean anything when you have a get out of jail free card because you are doing an officially-sanctioned penetration test. Except that after the penetration test Ira Winkler’s team discovered a physical \"Chinese intelligence operation in the middle of this small town, directly across the street from the research and development center of a Global 5 company\".\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "For every hack we hear about there are many hacks we don’t hear about. When you break into an adversary’s computer network the first thing you want to do is establish persistence. Most of those hacks we don’t hear about probably establish persistence. I would be surprised if China and the United States hadn’t established persistence in most of each others’ critical systems.\n", + "\n", + "In the event of a hot war between the United States and China, both sides will burn most of their zero-days immediately to cause as much disruption to the enemy as possible. It takes a lot of work to clean a hacker out of one of your systems. The cyber onslaught will probably overwhelm both sides’ ability to reset their software. They will have to focus on the most critical systems of all: communications.\n", + "\n", + "I expect all but the most secure systems (think \"US president’s personal phone\") will be entirely compromised. However, there are many ways to communicate. Both sides can improvise. Since secondary channels abound, it might be better just to spy on enemy communications instead of breaking them.\n", + "\n", + "It is theoretically possible to take control of enemy weapon systems too but I don’t think this will have a major impact. Weapon systems will continue to have human beings in the loop. Human beings can’t be hacked the way computers can.\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "I think compromised weapons systems will just be taken out of commission rather than commandeered. Some of them might be destroyed, but I think most will just be rendered temporarily inoperable.\n", + "\n", + "# Space War\n", + "\n", + "Satellites serve four purposes: reconnaissance, communication, navigation (GPS) and destroying other satellites. GPS going down would inconvenience both sides but it wouldn’t be decisive. There are other ways of navigating. The same goes for satellite communication.\n", + "\n", + "The most important use of satellites is reconnaissance. Aerial drones are nice but they can’t see as much at once as a camera in outer space. If both sides are restrained in their use of nuclear weapons then they might also be restrained in space warfare. That seems overly-optimistic to me. The primary objective of both sides will be to preserve their spy satellite capability while destroying the enemy’s.\n", + "\n", + "The price of space travel is going down. We can expect a large increase in the number of satellites between now and 2050, including spy satellites and anti-satellite satellites. It is plausible that satellite warfare could trigger Kessler syndrome in low earth orbit where collisions cascade into a giant mess of debris.\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "# Protracted Total War is Unlikely The initial exchange of missiles and zero-days will be fast. Critical tactical decisions may occur in the first few hours. The decisive fighting could be over in a matter of days. The limiting factor isn’t technical. It’s the speed at which leaders can make decisions. After a few weeks, both armies will be running out of missiles[1]. Both civilian populations will have suffered massive damage from cyberattacks on their civilian infrastructure. If satellite warfare triggers Kessler syndrome then much of the world’s communication infrastructure could be irreversibly damaged. The global economy would be a mess. Economic chaos would be bad for US interests and even worse for Chinese interests. If we do see a protracted war then it matters a lot what countries get involved. A conflict where India or Russia joins the fray is very different from a conflict where they don’t. Whatever happens, the tech level will probably go down. Advanced weapon systems like stealth drones take a long time to build. Destroy a few major semiconductor fabricators and the whole world runs low. But it’s hard for me to imagine a protracted war on the scale of World War II. In the past, getting your cities bombed was a small price to pay in order to expand your territory. China may sieze some already disputed territory like Taiwan or Kashmir. But it doesn’t want to administer a large empire. China’s primary objective is to maintain domestic stability.\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "Annexing Afghanistan or Vietnam or Pakistan would make China harder to govern, not easier.\n", + "\n", + "China’s secondary objective is to secure access to resources and markets. Destroying the United States might help tear down the LWO, but if China dragged itself down alongside the United States then that would just open up a power vacuum for countries like Brazil, India and South Africa.\n", + "\n", + "The United States isn’t an expansionary power either. America likes the LWO because the LWO supports American interests. I don’t see America sacrificing its own hegemony to preserve the LWO.\n", + "\n", + "The Seven Years War happened because Britain and France were expansionary powers. World War One happened because the European powers were expansionary. World War Two happened because Germany, Italy and Japan were expansionary powers. The United States and China aren’t expansionary powers.\n", + "\n", + "Instead of a protracted total war, we’ll probably see a ceasefire or deescalation. Besides control over Taiwan, the most important thing to come out of a conflict like this is a (re)establishment of the world order. A war would clarify who is and isn’t a superpower (anymore).\n", + "\n", + "\n", + "\n", + "\n", + " - If things go longer then it would be because the United States’ military force is spread around the world. Its reserve forces could take weeks just to get to China. ↩︎\n", + "\n", + " - Title: [Prediction] What war between the USA and China would look like in 2050, Author: lsusr, Date published: 2021-05-26, URL: https://www.lesswrong.com/posts/BY5f7iEzHtEDJLXS7/prediction-what-war-between-the-usa-and-china-would-look\n", + "\n", + "*Note: working on a research agenda, hence the large amount of small individual posts, to have things to link to in the main documents.*\n", + "\n", + "In my quest to synthesise human preferences, I’ve occasionally been asked whether I distinguish moral preferences from other types of preferences—for example, whether preferences for Abba or Beethoven, or avocado or sausages, should rank as high as human rights or freedom of speech.\n", + "\n", + "The answer is, of course not. But these are not the sort of things that should be built into the system by hand. This should be reflected in the meta-preferences. We label certain preferences \"moral\", and we often have the belief that these should have priority, to some extent, over merely \"selfish\" preferences (the extent of this belief varies from person to person, of course).\n", + "\n", + "I deliberately wrote the wrong word there for this formalism—we don’t have the \"belief\" that moral preferences are more important, we have the meta-preference that a certain class of beliefs, labelled \"moral\", whatever that turns out to mean, should be given greater weight. This is especially the case as there are a lot of cases where it is very unclear if a preference is moral or not (many people have strong moral-ish preferences over mainstream cultural and entertainment choices).\n", + " - Title: \"Moral\" as a preference label, Author: Stuart_Armstrong, Date published: 2019-03-26, URL: https://www.lesswrong.com/posts/BQzPyBjahycCN24dR/moral-as-a-preference-label\n", + "\n", + "This is an example of the sort of challenges that a preference synthesis process should be able to figure out on its own. If the method needs to be constantly tweaked to get over every small problem of definition, then it cannot work. As always, however, it need not get everything exactly right; indeed, it needs to be robust enough that it doesn’t change much if a borderline meta-preference such as \"everyone should know their own history\" gets labelled as moral or not.\n", + "\n", + " - Title: \"Moral\" as a preference label, Author: Stuart_Armstrong, Date published: 2019-03-26, URL: https://www.lesswrong.com/posts/BQzPyBjahycCN24dR/moral-as-a-preference-label\n", + "\n", + "http://​​thepiratebay.org/​​torrent/​​6554331/​​Papers_from_Philosophical_Transactions_of_the_Royal_Society__fro\n", + "\n", + "Greg Maxwell is torrenting 33Gib of public domain JSTOR documents that were behind paywalls.\n", + "\n", + "What’s your take on this, ethically, legally, etc?\n", + "\n", + "ETA: More on this: http://​​gigaom.com/​​2011/​​07/​​21/​​pirate-bay-jstor/​​\n", + "\n", + " - Title: Free (old) scientific papers [Link], Author: JackEmpty, Date published: 2011-07-21, URL: https://www.lesswrong.com/posts/LDnqfscCw78zibovh/free-old-scientific-papers-link\n", + "\n", + "Imagine you’re a self-modifying intelligent agent, and your mother is taking you out to lunch. Over coffee, she offers you a deal: if you will self-modify to be repulsed by eating animals to the extent that you become a vegetarian, she will pay for your lunch. Don’t try to cheat—she can tell when you’re lying. So you compare the total utility of changing with the total utility of not-changing, and you decide that you would rather continue to eat meat than have your lunch paid for. This is an example of an algorithm-dependent problem, a more general type than causal problems or Newcomb-like problems. Someone can see inside your brain to some extent and rewards you based not just on your actions or your decisions, but on your values or your patterns of thought. It’s sort of interesting, but they seem unlikely to happen because it’s difficult for us to modify our own algorithms and it’s difficult for some external observer to verify that we’ve really done so. AIs, on the other hand, can fulfill both of these requirements, and so might run into them on rare occasions. - More difficult modifications: The vegetarian/​omnivore tradeoff seems fairly comprehensible because of its nice properties. It’s fairly time-symmetrical since being a vegetarian is pretty much the same day to day. It’s familiar to us so that we don’t have to guess too much about what it’s like being a vegetarian.\n", + " - Title: Algorithm-dependent problems with self-modification, Author: Manfred, Date published: 2011-05-23, URL: https://www.lesswrong.com/posts/JcwhtxbNtofyJhK3B/algorithm-dependent-problems-with-self-modification\n", + "\n", + "And since we don’t feel like wanting to eat animals is an inherently valuable belief, we can just evaluate the utility of the consequences.\n", + "\n", + "What sort of modifications would be trickier? Well, going back to that last point, do we have inherently valuable beliefs? I’d argue that I do—I would not want to want to kill people even if God promised to keep an eye on me and stop me before I made any outward sign of trying. But it’s fairly simple to extend our utility function over our own brains.\n", + "\n", + "Worse is when you have multiple modifications. If you modified to be a vegetarian, and then modified to enjoy skydiving, it wouldn’t be a big deal—those are mostly independent. But what if your mom wanted you to modify something that impacted your self-modification system?\n", + "\n", + "\n", + "\n", + " - So:\n", + "\n", + "I think it’s not too hard to take an algorithm-determined problem, even with self-modification, and find the optimal classes of algorithms. This is analogous to the decisions made by a decision-theory-following agent. What would be nice is to have something analogous to a decision theory - \n", + "\n", + " - Title: Algorithm-dependent problems with self-modification, Author: Manfred, Date published: 2011-05-23, URL: https://www.lesswrong.com/posts/JcwhtxbNtofyJhK3B/algorithm-dependent-problems-with-self-modification\n", + "\n", + "***SUMMARY**: **Let’s collect people who want to work on for-profit companies that have significant positive impacts on many people’s lives.* Google provides a huge service to the world—efficient search of a vast amount of data. I would really like to see more for-profit businesses like Google, especially in underserved areas like those explored by non-profits GiveWell, Singularity Institute and CFAR. GiveWell is a nonprofit that is both working toward making humanity better, and thinking about leverage. Instead of hacking away at one branch of the problem of effective charity by working on one avenue for helping people, they’ve taken it meta. They’re providing a huge service by helping people choose non-profits to donate to that give the most bang for your buck, and they’re giving the non-profits feedback on how they can improve. I would love to see more problems taken meta like that, where people invest in high leverage things. Beyond these non-profits, I think there is a huge amount of low-hanging fruit for creating businesses that create a lot of good for humanity and make money. For-profit businesses that pay their employees and investors well have the advantage that they can entice very successful and comfortable people away from other jobs that are less beneficial to humanity.\n", + " - Title: Who Wants To Start An Important Startup?, Author: ShannonFriedman, Date published: 2012-08-16, URL: https://www.lesswrong.com/posts/YbCc3NRrr5avvWSHT/who-wants-to-start-an-important-startup\n", + "\n", + "Unlike non-profits where people are often trying to scrape by, doing the good of their hearts, people doing for-profits can live easy lives with luxurious self care while improving the world at the same time.\n", + "\n", + "It’s all well and good to appeal to altruistic motives, but a lot more people can be mobilzed if they don’t have to sacrifice their own comfort. I have learned a great deal about this from Jesse and Sharla at Rejuvenate. They train coaches and holistic practitioners in sales and marketing—enabling thousands of people to start businesses who are doing the sorts of things that advance their mission. They do this while also being multi-millionaires themselves, and maintaining a very comfortable lifestyle, taking the time for self-care and relaxation to recharge from long workdays.\n", + "\n", + "Less Wrong is read by thousands of people, many of whom are brilliant and talented. In addition, Less Wrong readers include people who are interested in the future of the world and think about the big picture. They think about things like AI and the vast positive and negative consequences it could have. In general, they consider possibilities that are outside of their immediate sensory experience.\n", + "\n", + "I’ve run into a lot of people in this community with some really cool, unique, and interesting ideas, for high-impact ways to improve the world. I’ve also run into a lot of talent in this community, and I have concluded that we have the resources to implement a lot of these same ideas.\n", + " - Title: Who Wants To Start An Important Startup?, Author: ShannonFriedman, Date published: 2012-08-16, URL: https://www.lesswrong.com/posts/YbCc3NRrr5avvWSHT/who-wants-to-start-an-important-startup\n", + "\n", + "Thus, I am opening up this post as a discussion for these possibilities. I believe that we can share and refine them on this blog, and that there are talented people who will execute them if we come up with something good. For instance, I have run into countless programmers who would love to be working on something more inspiring than what they’re doing now. I’ve also personally talked to several smart organizational leader types, such as Jolly and Evelyn, who are interested in helping with and/​or leading inspiring projects And that’s only the people I’ve met personally; I know there are a lot more folks like that, and people with talents and resources that haven’t even occurred to me, who are going to be reading this.\n", + "\n", + " **Topics to consider when examining an idea: **\n", + "\n", + "\n", + " - Tradeoffs between optimizing for good effects on the world v. making a profit.\n", + "\n", + " - Ways to improve both profitability and good effects on the world.\n", + "\n", + " - Timespan—projects for 3 months, 1 year, 5 years, 10+ years\n", + "\n", + " - Using resources efficiently (e.g. creating betting markets where a lot of people give opinions that they have enough confidence in to back with money, instead of having one individual trying to figure out probabilities)\n", + " - Title: Who Wants To Start An Important Startup?, Author: ShannonFriedman, Date published: 2012-08-16, URL: https://www.lesswrong.com/posts/YbCc3NRrr5avvWSHT/who-wants-to-start-an-important-startup\n", + "\n", + " - Opportunities for uber-programmers who can do anything quickly (they are reading and you just might interest and inspire them)\n", + "\n", + " - Opportunities for newbies trying to get a foot in the door who will work for cheap\n", + "\n", + " - What people/​resources do we have at our disposal now, and what can we do with that?\n", + "\n", + " - What people/​resources are still needed?\n", + "\n", + " - If you think of something else, make a comment about it in the thread for that, and it might get added to this list. \n", + "\n", + " An example idea from Reichart Von Wolfsheild:\n", + "\n", + "*A project to document the best advice we can muster into a single tome. It would inherently be something dynamic, that would grow and cover the topics important to humans that they normally seek refuge and comfort for in religion. A \"bible\" of sorts for the critical mind.*\n", + "\n", + "* Before things like wikis, this was a difficult problem to take on. But, that has changed, and the best information we have available can in fact be filtered for, and simplified. The trick now, is to organize it in a way that helps humans. which is not how most information is organized.*\n", + "\n", + "**Collaboration**\n", + "\n", + "\n", + " - **Please keep the mission in mind** (let’s have more for-profit companies working on goals that benefit people too!) when giving feedback. When you write a comment, consider whether it is contributing to that goal, or if it’s counterproductive to motivation or idea-generation, and edit accordingly. \n", + " - Title: Who Wants To Start An Important Startup?, Author: ShannonFriedman, Date published: 2012-08-16, URL: https://www.lesswrong.com/posts/YbCc3NRrr5avvWSHT/who-wants-to-start-an-important-startup\n", + "\n", + " - **Give feedback, the more specific the better.** Negative feedback is valuable because it tells us where to concentrate further work. It can also be a motivation-killer; it feels like punishment, and not just for the specific item criticized, so be charitable about the motives and intelligence of others, and stay mindful of how much and how aggressively you dole critiques out. (Do give critiques, they’re essential—just be gentle!) Also, distribute positive feedback for the opposite effect. More detail on giving the best possible feedback in this comment. - ** Please point other people with resources such as business experience, intelligence, implementation skills, and funding capacity at this post.** The more people with these resources who look at this and collaborate in the comments, the more likely it is for these ideas to get implemented. In addition to posting this to Less Wrong, I will be sending the link to a lot of friends with shrewd business skills, resources and talent, who might be interested in helping make projects happen, or possibly in finding people to work on their own projects since many of them are already working on projects to make the world better. - **Please provide feedback**. If anything good happens in your life as a result of this post or discussion, please comment about it and/​or give me feedback. It inspires people, and I have bets going that I’d like to win. Consider making bets of your own!\n", + " - Title: Who Wants To Start An Important Startup?, Author: ShannonFriedman, Date published: 2012-08-16, URL: https://www.lesswrong.com/posts/YbCc3NRrr5avvWSHT/who-wants-to-start-an-important-startup\n", + "\n", + "It is also important to let me know if you are going to use the ideas, so that we don’t end up with needless duplication and competition.\n", + "\n", + "*Finally: If this works right, there will be lots of information flying around. Check out the organization thread and the wiki.*****\n", + "\n", + " - Title: Who Wants To Start An Important Startup?, Author: ShannonFriedman, Date published: 2012-08-16, URL: https://www.lesswrong.com/posts/YbCc3NRrr5avvWSHT/who-wants-to-start-an-important-startup\n", + "\n", + "Contents - Core Debate - Recursive Self-Improvement - Continuous Progress on Benchmarks *Linkpost for \"**Yudkowsky Contra Christiano on AI Takeoff Speeds**\", published by Scott Alexander on March 4, 2021. Here is the **original conversation**. Cross-posted on the EA Forum **here**. * Key Quotes: > Around the end of last year, Paul and Eliezer had a complicated, protracted, and indirect debate, culminating in a few hours on the same Discord channel. Although the real story is scattered over several blog posts and chat logs, I’m going to summarize it as if it all happened at once. ## Core Debate Paul Christiano’s position: > Paul sums up his half of the debate as: \"There will be a complete 4 year interval in which world output doubles, before the first 1 year interval in which world output doubles. (Similarly, we’ll see an 8 year doubling before a 2 year doubling, etc. )\" That is—if any of this \"transformative AI revolution\" stuff is right at all, then at some point GDP is going to go crazy (even if it’s just GDP as measured by AIs, after humans have been wiped out). Paul thinks it will go crazy slowly. Right now world GDP doubles every ~25 years. Paul thinks it will go through an intermediate phase (doubles within 4 years) before it gets to a truly crazy phase (doubles within 1 year). Why? Partly based on common sense. Whenever you can build a cool thing at time T, probably you could build a slightly less cool version at time T-1.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "And slightly less cool versions of cool things are still pretty cool, so there shouldn’t be many cases where a completely new and transformative thing starts existing without any meaningful precursors. But also because this is how everything always works. [See explanations from Katja Grace and Nintil. ] Eliezer Yudkowsky’s response: > Eliezer counters that although progress may retroactively look gradual and continuous when you know what metric to graph it on, it doesn’t necessarily look that way in real life by the measures that real people care about. (one way to think of this: imagine that an AI’s effective IQ starts at 0.1 points, and triples every year, but that we can only measure this vaguely and indirectly. The year it goes from 5 to 15, you get a paper in a third-tier journal reporting that it seems to be improving on some benchmark. The year it goes from 66 to 200, you get a total transformation of everything in society. But later, once we identify the right metric, it was just the same rate of gradual progress the whole time. ) So Eliezer is much less impressed by the history of previous technologies than Paul is. He’s also skeptical of the \"GDP will double in 4 years before it doubles in 1\" claim, because of two contingent disagreements and two fundamental disagreements. The first contingent disagreement: government regulations make it hard to deploy imperfect things, and non-trivial to deploy things even after they’re perfect.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "Eliezer has non-jokingly said he thinks AI might destroy the world before the average person can buy a self-driving car. Why? Because the government has to approve self-driving cars (and can drag its feet on that), but the apocalypse can happen even without government approval. In Paul’s model, sometime long before superintelligence we should have AIs that can drive cars, and that increases GDP and contributes to a general sense that exciting things are going on. Eliezer says: fine, what if that’s true? Who cares if self-driving cars will be practical a few years before the world is destroyed? It’ll take longer than that to lobby the government to allow them on the road. The second contingent disagreement: superintelligent AIs can lie to us. Suppose you have an AI which wants to destroy humanity, whose IQ is doubling every six months. Right now it’s at IQ 200, and it suspects that it would take IQ 800 to build a human-destroying superweapon. Its best strategy is to lie low for a year. If it expects humans would turn it off if they knew how close it was to superweapons, it can pretend to be less intelligent than it really is. The period when AIs are holding back so we don’t discover their true power level looks like a period of lower-than-expected GDP growth—followed by a sudden FOOM once the AI gets its superweapon and doesn’t need to hold back.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "So *even if Paul is conceptually right* and fundamental progress proceeds along a nice smooth curve, it might not look to us like a nice smooth curve, because regulations and deceptive AIs could prevent mildly-transformative AI progress from showing up on graphs, but wouldn’t prevent the extreme kind of AI progress that leads to apocalypse. To an outside observer, it would just look like nothing much changed, nothing much changed, nothing much changed, and then suddenly, FOOM. But even aside from this, Eliezer doesn’t think Paul is conceptually right! He thinks that *even on the fundamental level*, AI progress is going to be discontinuous. It’s like a nuclear bomb. Either you don’t have a nuclear bomb yet, or you do have one and the world is forever transformed. There is a specific moment at which you go from \"no nuke\" to \"nuke\" without any kind of \"slightly worse nuke\" acting as a harbinger. He uses the example of chimps → humans. Evolution has spent hundreds of millions of years evolving brainier and brainier animals (not teleologically, of course, but in practice). For most of those hundreds of millions of years, that meant the animal could have slightly more instincts, or a better memory, or some other change that still stayed within the basic animal paradigm. At the chimp → human transition, we suddenly got tool use, language use, abstract thought, mathematics, swords, guns, nuclear bombs, spaceships, and a bunch of other stuff.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "The rhesus monkey → chimp transition and the chimp → human transition both involved the same ~quadrupling of neuron number, but the former was pretty boring and the latter unlocked enough new capabilities to easily conquer the world. The GPT-2 → GPT-3 transition involved centupling parameter count. Maybe we will keep centupling parameter count every few years, and most times it will be incremental improvement, and one time it will conquer the world. But even talking about centupling parameter points is giving Paul too much credit. Lots of past inventions didn’t come by quadrupling or centupling something, they came by discovering \"the secret sauce\". The Wright brothers (he argues) didn’t make a plane with 4x the wingspan of the last plane that didn’t work, they *invented the first plane that could fly at all. * The Hiroshima bomb wasn’t some previous bomb but bigger, it was what happened after a lot of scientists spent a long time thinking about a fundamentally different paradigm of bomb-making and brought it to a point where it could work at all. The first transformative AI isn’t going to be GPT-3 with more parameters, it will be what happens after someone discovers how to make machines truly intelligent. (this is the same debate Eliezer had with Ajeya over the Biological Anchors post; have I mentioned that Ajeya and Paul are married? ) ## Recursive Self-Improvement Yudkowsky on recursive self-improvement: > This is where I think Eliezer most wants to take the discussion.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "The idea is: once AI is smarter than humans, it can do a superhuman job of developing new AI. In his Microeconomics paper, he writes about an argument he (semi-hypothetically) had with Ray Kurzweil about Moore’s Law. Kurzweil expected Moore’s Law to continue forever, even after the development of superintelligence. Eliezer objects: \"Suppose we were dealing with minds running a million times as fast as a human, at which rate they could do a year of internal thinking in thirty-one seconds, such that the total subjective time from the birth of Socrates to the death of Turingwould pass in 20.9 hours. Do you still think the best estimate for how longit would take them to produce their next generation of computing hardwarewould be 1.5 orbits of the Earth around the Sun\"? That is: the fact that it took 1.5 years for transistor density to double isn’t a natural law. It’s *pointing to* a law that the amount of resources (most notably intelligence) that civilization focused on the transistor-densifying problem equalled the amount it takes to double it every 1.5 years. If some shock drastically changed available resources (by eg speeding up human minds a million times), this would change the resources involved, and the same laws would predict transistor speed doubling in some shorter amount of time (naively 0.000015 years, although realistically at that scale other inputs would dominate).\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "So when Paul derives clean laws of economics showing that things move along slow growth curves, Eliezer asks: why do you think they would keep doing this when one of the discoveries they make along that curve might be \"speeding up intelligence a million times\"? (Eliezer actually thinks improvements in the quality of intelligence will dominate improvements in speed—AIs will mostly be smarter, not just faster—but speed is a useful example here and we’ll stick with it) Christiano’s response on recursive self-improvement: > *Summary of my response: Before there is AI that is great at self-improvement there will be AI that is mediocre at self-improvement. * Powerful AI can be used to develop better AI (amongst other things). This will lead to runaway growth. This on its own is not an argument for discontinuity: before we have AI that radically accelerates AI development, the slow takeoff argument suggests we will have AI that *significantly* accelerates AI development (and before that, *slightly *accelerates development). That is, an AI is just another, faster step in the hyperbolic growth we are currently experiencing, which corresponds to a further increase in rate but not a discontinuity (or even a discontinuity in rate).\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "The most common argument for recursive self-improvement introducing a new discontinuity seems be: some systems \"fizzle out\" when they try to design a better AI, generating a few improvements before running out of steam, while others are able to autonomously generate more and more improvements. This is basically the same as the universality argument in a previous section. ## Continuous Progress on Benchmarks Matthew Barnett’s response highlights relatively continuous progress by language models on the Penn Treebank benchmark, sparking three responses. First, from Gwern: > The impact of GPT-3 had nothing whatsoever to do with its perplexity on Penn Treebank . . . the impact of GPT-3 was in establishing that trendlines did continue in a way that shocked pretty much everyone who’d written off ‘naive’ scaling strategies. Progress is made out of stacked sigmoids: if the next sigmoid doesn’t show up, *progress doesn’t happen*. Trends happen, until they stop. Trendlines are not caused by the laws of physics. You can dismiss AlphaGo by saying \"oh, that just continues the trendline in ELO I just drew based on MCTS bots\", but the fact remains that MCTS progress had stagnated, and here we are in 2021, and pure MCTS approaches do not approach human champions, much less beat them. Appealing to trendlines is roughly as informative as \"calories in calories out\"; ‘the trend continued because the trend continued’. A new sigmoid being discovered is extremely important.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "> GPT-3 further showed completely unpredicted emergence of capabilities across *downstream* tasks which are not measured in PTB perplexity. There is nothing obvious about a PTB BPC of 0.80 that causes it to be useful where 0.90 is largely useless and 0.95 is a laughable toy. (OAers may have had faith in scaling, but they could not have told you in 2015 that interesting behavior would start at 𝒪(1b), and it’d get really cool at 𝒪(100b). ) That’s why it’s such a useless metric. There’s only one thing that a PTB perplexity can tell you, under the pretraining paradigm: when you have reached human AGI level. (Which is useless for obvious reasons: much like saying that \"if you hear the revolver click, the bullet wasn’t in that chamber and it was safe\". Surely true, but a bit late. ) It tells you nothing about intermediate levels. I’m reminded of the Steven Kaas line: \"Why idly theorize when you can JUST CHECK and find out the ACTUAL ANSWER to a superficially similar-sounding question SCIENTIFICALLY\"? Then from Yudkowsky: > What does it even mean to be a gradualist about any of the important questions like [the ones Gwern mentions], when they don’t relate in known ways to the trend lines that are smooth? Isn’t this sort of a shell game where our surface capabilities do weird jumpy things, we can point to some trend lines that were nonetheless smooth, and then the shells are swapped and we’re told to expect gradualist AGI surface stuff?\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "This is part of the idea that I’m referring to when I say that, even as the world ends, maybe there’ll be a bunch of smooth trendlines underneath it that somebody could look back and point out. (Which you could in fact have used to predict all the key jumpy surface thresholds, *if *you’d watched it all happen on a few other planets and had any idea of where jumpy surface events were located on the smooth trendlines—but we haven’t watched it happen on other planets so the trends don’t tell us much we want to know. ) And Christiano: > [Yudkowsky’s argument above] seems totally bogus to me. It feels to me like you mostly don’t have views about the actual impact of AI as measured by jobs that it does or the $s people pay for them, or performance on *any* benchmarks that we are currently measuring, while I’m saying I’m totally happy to use gradualist metrics to predict any of those things. If you want to say \"what does it mean to be a gradualist\" I can just give you predictions on them. To you this seems reasonable, because e.g. $ and benchmarks are not the right way to measure the kinds of impacts we care about. That’s fine, you can propose something other than $ or measurable benchmarks. If you can’t propose anything, I’m skeptical.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "On the question of why AI takeoff speeds matter in the first place: > If there’s a slow takeoff (ie gradual exponential curve), it will become obvious that some kind of terrifying transformative AI revolution is happening, before the situation gets apocalyptic. There will be time to prepare, to test slightly-below-human AIs and see how they respond, to get governments and other stakeholders on board. We don’t have to get every single thing right ahead of time. On the other hand, because this is proceeding along the usual channels, it will be the usual variety of muddled and hard-to-control. With the exception of a few big actors like the US and Chinese government, and maybe the biggest corporations like Google, the outcome will be determined less by any one agent, and more by the usual multi-agent dynamics of political and economic competition. There will be lots of opportunities to affect things, but no real locus of control to do the affecting. If there’s a fast takeoff (ie sudden FOOM), there won’t be much warning. Conventional wisdom will still say that transformative AI is thirty years away. All the necessary pieces (ie AI alignment theory) will have to be ready ahead of time, prepared blindly without any experimental trial-and-error, to load into the AI as soon as it exists. On the plus side, a single actor (whoever has this first AI) will have complete control over the process.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "If this actor is smart (and presumably they’re a little smart, or they wouldn’t be the first team to invent transformative AI), they can do everything right without going through the usual government-lobbying channels. So the slower a takeoff you expect, the less you should be focusing on getting every technical detail right ahead of time, and the more you should be working on building the capacity to steer government and corporate policy to direct an incoming slew of new technologies. Also Raemon on AI safety in a soft-takeoff world: > I totally think there are people who sort of nod along with Paul, using it as an excuse to believe in a rosier world where things are more comprehensible and they can imagine themselves doing useful things without having a plan for solving the actual hard problems. Those types of people exist. I think there’s some important work to be done in confronting them with the hard problem at hand. But, also… Paul’s world AFAICT *isn’t actually rosier*. It’s potentially *more* frightening to me. In Smooth Takeoff world, you can’t carefully plan your pivotal act with an assumption that the strategic landscape will remain roughly the same by the time you’re able to execute on it. Surprising partial-gameboard-changing things could happen that affect what sort of actions are tractable. Also, dumb, boring ML systems run amok could kill everyone before we even get to the part where recursive self improving consequentialists eradicate everyone.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "I think there is still something seductive about this world – dumb, boring ML systems run amok *feels like* the sort of problem that is easier to reason about and maybe solve. (I don’t think it’s *actually* necessarily easier to solve, but I think it can feel that way, whether it’s easier or not). And if you solve ML-run-amok-problems, you still end up dead from recursive-self-improving-consequentialists if you didn’t have a plan for them. Metaculus has a question about AI takeoff speeds: > This is the Metaculus forecasting question corresponding to Paul’s preferred formulation of hard/​soft takeoff. Metaculans think there’s a 69% chance it’s true. But it fell by about 4% after the debate, suggesting that some people got won over to Eliezer’s point of view. Conclusions from the debate: > They both agreed they weren’t going to resolve this today, and that the most virtuous course would be to generate testable predictions on what the next five years would be like, in the hopes that one of their models would prove obviously much more productive at this task than the other. But getting these predictions proved harder than expected. Paul believes \"everything will grow at a nice steady rate\" and Eliezer believes \"everything will grow at a nice steady rate until we suddenly die\", and these worlds look the same until you are dead. I am happy to report that three months later, the two of them finally found an empirical question they disagreed on and made a bet on it.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "The difference is: Eliezer thinks AI is a little bit more likely to win the International Mathematical Olympiad before 2025 than Paul (under a specific definition of \"win\").\n", + "\n", + "I haven’t followed the many many comment sub-branches it would take to figure out how that connects to any of this, but if it happens, update a little towards Eliezer, I guess.\n", + "\n", + "Also this: > I didn’t realize this until talking to Paul, but \"holder of the gradualist torch\" is a relative position—Paul still thinks there’s about a 1⁄3 chance of a fast takeoff.\n", + " - Title: Yudkowsky Contra Christiano on AI Takeoff Speeds [Linkpost], Author: aogara, Date published: 2022-04-05, URL: https://www.lesswrong.com/posts/2S4KF7LmgPTXr9AgF/yudkowsky-contra-christiano-on-ai-takeoff-speeds-linkpost\n", + "\n", + "Contents\n", + " - Top Results\n", + " - Top Reviewers\n", + " - Complete Results (1000+ Karma)\n", + " - What does this mean, and what happens now? \n", + "*You can see the full voting results here: **1000+ karma voters** (**All voters*)\n", + "The 2019 Review votes are in!\n", + "This year, 88 voters participated, evaluating 116 posts. (Of those voters, 61 had 1000+ karma, and will be weighted more highly in the moderation team’s decision of what to include in the Best of 2019 Books)\n", + "The LessWrong Moderation team will be reflecting on these results and using them as a major input into \"what to include in the 2019 books.\" \n", + "# Top Results\n", + "The top 15 results from the 1000+ karma users are:\n", + "\n", + " - **What failure looks like** by Paul Christiano\n", + "\n", + " - **Risks from Learned Optimization: Introduction**, by evhub, Chris van Merwijk, vlad_m, Joar Skalse and Scott Garrabrant\n", + "\n", + " - **The Parable of Predict-O-Matic**, by Abram Demski\n", + "\n", + " - **Book Review: The Secret Of Our Success**, by Scott Alexander\n", + "\n", + " - **Being the (Pareto) Best in the World**, by Johnswentworth\n", + "\n", + " - **Rule Thinkers In, Not Out**, by Scott Alexander\n", + "\n", + " - **Book summary: Unlocking the Emotional Brain**, by Kaj Sotala\n", + "\n", + " - **Asymmetric Justice**, by Zvi Mowshowitz\n", + "\n", + " - **Heads I Win, Tails?—Never Heard of Her; Or, Selective Reporting and the Tragedy of the Green Rationalists**, by Zack Davis\n", + " - Title: 2019 Review: Voting Results!, Author: Raemon, Date published: 2021-02-01, URL: https://www.lesswrong.com/posts/kdGSTBj3NA2Go3XaE/2019-review-voting-results\n", + "\n", + " - **1960: The Year The Singularity Was Cancelled**, by Scott Alexander\n", + "\n", + " - **Selection vs Control**, by Abram Demski\n", + "\n", + " - **You Have About Five Words**, by Raymond Arnold\n", + "\n", + " - **The Schelling Choice is \"Rabbit\", not \"Stag\"**, by Raymond Arnold\n", + "\n", + " - **Noticing Frame Differences**, by Raymond Arnold\n", + "\n", + " - **\"Yes Requires the Possibility of No\"**, by Scott Garrabrant\n", + "# Top Reviewers\n", + "Meanwhile, we also had a lot of great reviews. One of the most valuable things I found about the review process was that it looks at *lots* of great posts at once, which led me to find connections between them I had previously missed. We’ll be doing a more in-depth review of the best reviews later on, but for now, I wanted to shoutout to the people who did a bunch of great review work. \n", + "The top reviewers (aggregating the total karma of their review-comments) were:\n", + "Some things I particularly appreciated were: \n", + "\n", + " - johnswentworth, Zvi and others providing fairly comprehensive reviews of many different posts, taking stock of how some posts fit together.\n", + "\n", + " - Jacobjacob and magfrump who stuck out in my mind for doing particularly \"epistemic spot check\" type reviews, which are often more effortful.\n", + "# Complete Results (1000+ Karma)\n", + "*You can see the full voting results here: **1000+ karma voters** (**All voters**)*\n", + "To help users see the spread of the vote data, we’ve included *swarmplot** *visualizations.\n", + " - Title: 2019 Review: Voting Results!, Author: Raemon, Date published: 2021-02-01, URL: https://www.lesswrong.com/posts/kdGSTBj3NA2Go3XaE/2019-review-voting-results\n", + "\n", + " - Only votes with weights between −10 and 16 are plotted. Outliers are in the image captions. - Gridlines are spaced 2 points apart. - Concrete illustration: The plot immediately below has 18 votes ranging in strength from −3 to 12. **Rank****Title**Visualization**1****What failure looks like****2****Risks from Learned Optimization: Introduction****3****The Parable of Predict-O-Matic****4.5****Being the (Pareto) Best in the World****4.5****Book Review: The Secret Of Our Success****6****Rule Thinkers In, Not Out****7****Book summary: Unlocking the Emotional Brain**Outlier: +20**8.5****Asymmetric Justice****8.5****Heads I Win, Tails? R\n", + " - Title: 2019 Review: Voting Results!, Author: Raemon, Date published: 2021-02-01, URL: https://www.lesswrong.com/posts/kdGSTBj3NA2Go3XaE/2019-review-voting-results\n", + "\n", + "****40****Propagating Facts into Aesthetics****42****Gradient hacking****44****The Amish, and Strategic Norms around Technology****44****Power Buys You Distance From The Crime****44****Paper-Reading for Gears****48.5****How to Ignore Your Emotions (while also thinking you’re awesome at emotions)****48.5****The Real Rules Have No Exceptions****48.5****Coherent decisions imply consistent utilities****48.5****Alignment Research Field Guide****48.5****Blackmail****48.5****The Curse Of The Counterfactual****52.5****The Credit Assignment Problem****52.5****Reason isn’t magic****54****Mental Mountains****56.5****Simple Rules of Law****56.5****Utility ≠ Reward****56.5****Is Rationalist Self-Improvement Real? ****56.5****Literature Review: Distributed Teams****59****Steelmanning Divination****60****Book Review: Design Principles of Biological Circuits****61****Building up to an Internal Family Systems model****62****Evolution of Modularity****63****[Answer] Why wasn’t science invented in China? ****64****How Much is Your Time Worth? ****65.5****Book Review: The Structure Of Scientific Revolutions****65.5****Everybody Knows****68.5****Sequence introduction: non-agent and multiagent models of mind****68.5****From Personal to Prison Gangs: Enforcing Prosocial Behavior****68.5****Some Ways Coordination is Hard****68.5****Circle Games****71****Why Subagents? ****73.5****Healthy Competition****73.5****Where to Draw the Boundaries?\n", + " - Title: 2019 Review: Voting Results!, Author: Raemon, Date published: 2021-02-01, URL: https://www.lesswrong.com/posts/kdGSTBj3NA2Go3XaE/2019-review-voting-results\n", + "\n", + "****73.5****Does it become easier, or harder, for the world to coordinate around not building AGI as time goes on? ****73.5****Six AI Risk/​Strategy Ideas****76.5****Thoughts on Human Models****76.5****Classifying specification problems as variants of Goodhart’s Law****79****Book Summary: Consciousness and the Brain****79****Gears vs Behavior****79****Strategic implications of AIs’ ability to coordinate at low cost, for example by merging****82****Book Review: Secular Cycles****82****Some Thoughts on My Psychiatry Practice****82****Coordination Surveys: why we should survey to organize responsibilities, not just predictions****84.5****The Hard Work of Translation (Buddhism)****84.5****[Part 2] Amplifying generalist research via forecasting – results from a preliminary exploration****86.5****Soft takeoff can still lead to decisive strategic advantage****86.5****Total horse takeover****88.5****Complex Behavior from Simple (Sub)Agents****88.5****Less Competition, More Meritocracy? ****90.5****Megaproject management****90.5****But exactly how complex and fragile? ****92.5****Trauma, Meditation, and a Cool Scar****92.5****Turning air into bread****95****AlphaStar: Impressive for RL progress, not for AGI progress****95****Dishonest Update Reporting****95****What are the open problems in Human Rationality?\n", + " - Title: 2019 Review: Voting Results!, Author: Raemon, Date published: 2021-02-01, URL: https://www.lesswrong.com/posts/kdGSTBj3NA2Go3XaE/2019-review-voting-results\n", + "\n", + "****97.5****Integrating the Lindy Effect****97.5****mAIry’s room: AI reasoning to solve philosophical problems****99****S-Curves for Trend Forecasting****100.5****Partial summary of debate with Benquo and Jessicata [pt 1]****100.5****The Power to Teach Concepts Better****102.5****Instant stone (just add water!\n", + "\n", + ")****102.5****Autism And Intelligence: Much More Than You Wanted To Know****104****Relevance Norms; Or, Gricean Implicature Queers the Decoupling/​Contextualizing Binary****105****Rationality, Levels of Intervention, and Empiricism****106****The Zettelkasten Method****107****Two explanations for variation in human abilities****108****No, it’s not The Incentives—it’s you**Outlier: −11**109****Firming Up Not-Lying Around Its Edge-Cases Is Less Broadly Useful Than One Might Initially Think****110****The Power to Demolish Bad Arguments****111****Neural Annealing: Toward a Neural Theory of Everything (crosspost)**Outlier: −20**112****Dual Wielding****113****Approval Extraction Advertised as Production**Outlier: −15**114****The AI Timelines Scam**Outlier: −13**115****Debate on Instrumental Convergence between LeCun, Russell, Bengio, Zador, and More**# What does this mean, and what happens now?\n", + "\n", + " - Create common knowledge about how the LessWrong community feels about various posts and the progress we’ve made.\n", + " - Title: 2019 Review: Voting Results!, Author: Raemon, Date published: 2021-02-01, URL: https://www.lesswrong.com/posts/kdGSTBj3NA2Go3XaE/2019-review-voting-results\n", + "\n", + " - Improve our longterm incentives, feedback, and rewards for authors. - Help create a highly curated \"Best of 2019\" Sequence and Book. Over the next few months we will take the results of this vote and make it into another curated collection of essays, just as we did with last years results, which turned into the \"A Map that Reflects the Territory\" essay collection. Voting, review and nomination participation was substantially greater this year than last year (something between a 30% and 80% increase, depending on which metrics you look at), which makes me hopeful about this tradition living on as a core piece of infrastructure for LessWrong. I was worried that participation would fall off after the initial excitement of last year, but I am no longer as worried about that. Both this year and last year we have also seen little correlation with the vote results and the karma of the posts, which is an important sanity check I have for whether going through all the effort of this review is worth it. If the ranking was basically just the same as the karma scores of the post, then we wouldn’t be getting much information out of the review. But as it stands, I trust the results of this review much more than I would trust someone just pressing the \"sort by karma\" button on the all-posts page, and I think as the site and community continues to grow, the importance of the robustness of the review will only grow. Thank you all for participating in this year’s review.\n", + " - Title: 2019 Review: Voting Results!, Author: Raemon, Date published: 2021-02-01, URL: https://www.lesswrong.com/posts/kdGSTBj3NA2Go3XaE/2019-review-voting-results\n", + "\n", + "I am pleased with results, and brimming with ideas for the new set of books that I am looking forward to implementing, and I think the above is already a valuable resource if someone wants to decide how to best catch up with all the great writing here on the site.\n", + " - Title: 2019 Review: Voting Results!, Author: Raemon, Date published: 2021-02-01, URL: https://www.lesswrong.com/posts/kdGSTBj3NA2Go3XaE/2019-review-voting-results\n", + "\n", + "Reminder: London meetup on Sunday May 1 at 14:00 at the Shakespeares Head (official page) on Kingsway near Holborn Tube station. Note that there’s more than one pub in London with that name, so make sure you get the right one. As always, we’ll have a big picture of a paperclip on the table so you can find us; I look like this. Hope to see lots of you there!\n", + "\n", + " - Title: Reminder: London meetup, Sunday 2pm, near Holborn, Author: Paul Crowley, Date published: 2011-04-28, URL: https://www.lesswrong.com/posts/6zhgiDHdiNycGSAby/reminder-london-meetup-sunday-2pm-near-holborn\n", + "\n", + "15 Jan − 0 posts\n", + "\n", + "16 Jan − 1 post\n", + "\n", + "17 Jan − 1 post\n", + "\n", + "18 Jan − 1 post\n", + "\n", + "19 Jan − 0 posts\n", + "\n", + "20 Jan − 2 posts\n", + "\n", + "21 Jan − 0 posts\n", + "\n", + "22 Jan − 1 post\n", + "\n", + "23/​24/​25/​26/​27 Jan − 0 posts\n", + "\n", + "28 Jan − 1 post\n", + "\n", + "29 Jan − 1 post\n", + "\n", + "30 Jan − 0 posts\n", + "\n", + "31 Jan − 3 posts\n", + "\n", + "1 Feb − 2 posts\n", + "\n", + "2 Feb − 1 post\n", + "\n", + "3 Feb − 0 posts\n", + "\n", + "4 Feb − 1 post\n", + "\n", + "5 Feb − 3 posts\n", + "\n", + "Maybe 23-27 was Christmas? But I’ve gotten a general feeling that activity spikes around the same time. Perhaps when the site is populated with posts, people spend more time here, and then think more on related topics, and thus are more likely to post?\n", + "\n", + "Note that this is all posts, not just promoted posts. It also includes rationality meetups and quotes threads—maybe it’d be more interesting analysis without that… I definitely get the feeling that a thought provoking post generates more. Whereas inactivity generates more inactivity. Thoughts?\n", + "\n", + " - Title: A bit meta: Do posts come in batches? If so, why?, Author: lionhearted, Date published: 2011-02-06, URL: https://www.lesswrong.com/posts/MnftBAcPmvwZX6ib3/a-bit-meta-do-posts-come-in-batches-if-so-why\n", + "\n", + "*I was originally going to post this as a comment into the UFAI & great filter thread, but since I noticed that my comment didn’t include a single word of AIs I thought about making an entire new discussion thread and I continued writing to improve the quality from comment to post. ***The essay is intended as thought-provoking* and I don’t have the required knowledge in the related fields and I mostly pieced this together by browsing wikipedia, but hopefully it gets you thinking! * Personally I think when considering the Drake Equation it’s important to note that it actually took ridiculously long for intelligent life to evolve *here* and that we’re on a *finite timeline*. The drake equation contains the rate of star formation, the number of planets in the stars, it even has a variable for the time it takes for life to evolve to the point of signaling detectably into outer space, etc. but it’s also important to pay attention to that the average setup of the universe has changed. On earth life has existed for almost 4 billion years and it has only been 43 years since our civilization first visited the moon and ~1½ centuries since the invention of radio? That is a *very* small time frame. Particularly if we consider that **~4 billion years is between a quarter and a third of the age of the universe itself. **When we consider the Great Filter we can at least propose that there have been **several** mass extinction events which *failed *to end all life on earth.\n", + " - Title: Thoughts on the Drake Equation and the Great Filter, Author: [deleted], Date published: 2012-12-23, URL: https://www.lesswrong.com/posts/Bw7KekTwF6uBmcKaC/thoughts-on-the-drake-equation-and-the-great-filter\n", + "\n", + "I think it’s a valid argument to say that for an example any powerful impact could have ended all life or *reset* the evolution of life some/​any number of degrees—and it has been ~70 years since the initiation of the Manhattan project and already humanity has the potential to go through a thermonuclear war that could end human life on the planet, or rollback the game of life through nuclear winter. Mars could have been habitable. For an example there’s no liquid water on Mars now, though there should’ve been earlier. The habitable zone as theoretized is considerably narrow—For an example: **If at any point in the history of (life on) earth the average surface temperature had climbed to 200 celsius for whatever reason **I’m pretty sure that would’ve made our planet like all the other planets observed—so far—in that they don’t seem contain intelligent life. What I mean by this is that even though a vast number of planets reside in the habitable zone of some star, they have to maintain those conditions for a *very long time*, and that’s just one variable. Which by the way is a pretty important thing to note when talking about things such as the greenhouse effect for an example. Some people seem to have this idea of \"natural balance\" that occurs automatically. It’s as if those people are not looking at the \"natural balance\" on some of the other planets. *Where’s the mechanism anyway*? Milankovitch cycles?\n", + " - Title: Thoughts on the Drake Equation and the Great Filter, Author: [deleted], Date published: 2012-12-23, URL: https://www.lesswrong.com/posts/Bw7KekTwF6uBmcKaC/thoughts-on-the-drake-equation-and-the-great-filter\n", + "\n", + "Even algae managed to start an ice age according to some theories, humans certainly have the potential to do more harm than that and it’s not like we only have to care for extinction events that we brought upon ourselves.\n", + "\n", + "In addition to this it seems to me frequently neglected that the conditions inside the universe have changed considerably with the aging universe.\n", + "\n", + "Earth is not constantly bombarded by collisions, it takes time for stars and planets and so forth to attain their form, the average age of stars has changed.\n", + "\n", + "In other words the *habitability of the entire universe changes over time*, though not in a particularly synchronous fashion.\n", + "\n", + "If this does not seem reasonable then consider the following: Was the likelihood for finding intelligent life in any location of the universe when it was 1 billion years old the same as it is today?\n", + "\n", + "How about when the universe was 4 billion years old?\n", + "\n", + "8 billion years?\n", + "\n", + "Most stars are between 1 to 10 billion years of age according to wikipedia.\n", + "\n", + "Human species itself has gone through some sort of a bottleneck, a historical token worth reflecting upon: Had the event been worse and those few remaining members of our ancenstry perished the planet earth would arguably still be without intelligent civilizations even today.\n", + " - Title: Thoughts on the Drake Equation and the Great Filter, Author: [deleted], Date published: 2012-12-23, URL: https://www.lesswrong.com/posts/Bw7KekTwF6uBmcKaC/thoughts-on-the-drake-equation-and-the-great-filter\n", + "\n", + "**This line of reasoning in my opinion favors two different details:**\n", + "\n", + "1. Since our intelligence took almost 4 billion years to evolve, any event within that time that could’ve wiped out all the progress, would’ve occured before the rise of intelligent civilizations—and so all those events contribute to the Great Filter \n", + "\n", + "2. The often contemplated likelihood that human intelligence is among the earliest intelligent species to arise, if life had been considerably less likely in the earlier stages of the universe. (which is very complatible with the fact that we have not observed life elsewhere—or at least that’s somewhat complementary to likelihood of intelligent life) In otherwords if our species is within the first 5% of the intelligent civilizations to arise that should be reflected upon our observations. Of course the same is true for the last 5%, etc. This is an important point, because that’s not the kind of reliability science rests upon.\n", + "\n", + "Remember how life taking almost ~4 billion years to evolve on earth was a ~1/​3 (rather ~2/​7) of the age of the entire universe? Well our solar system is only 4.6 billion years old. Life on earth has been evolving practically since the formation of our solar system and at no point in time were all the replicators wiped out.\n", + " - Title: Thoughts on the Drake Equation and the Great Filter, Author: [deleted], Date published: 2012-12-23, URL: https://www.lesswrong.com/posts/Bw7KekTwF6uBmcKaC/thoughts-on-the-drake-equation-and-the-great-filter\n", + "\n", + "So, any thoughts?\n", + "\n", + " - Title: Thoughts on the Drake Equation and the Great Filter, Author: [deleted], Date published: 2012-12-23, URL: https://www.lesswrong.com/posts/Bw7KekTwF6uBmcKaC/thoughts-on-the-drake-equation-and-the-great-filter\n", + "\n", + "Link post\n", + "\n", + " - Title: Can anyone be rational and not vegan?, Author: Sophivorus, Date published: 2016-11-23, URL: https://www.lesswrong.com/posts/hasbts6BC9JjKNmGv/can-anyone-be-rational-and-not-vegan\n", + "\n", + "**This summary was posted to LW Main on August 12th. The following week’s summary is here.** New meetups (or meetups with a hiatus of more than a year) are happening in: \n", + " - Bay City Meetup: 19 August 2016 01:25PM - Munich Meetup in August: 13 August 2016 01:00PM \n", + "Irregularly scheduled Less Wrong meetups are taking place in: \n", + " - Australian-ish Online Hangout: 20 August 2016 07:30PM - Baltimore Area Weekly Meetup: 14 August 2016 08:00PM - European Community Weekend: 02 September 2016 03:35PM The remaining meetups take place in cities with regular scheduling, but involve a change in time or location, special meeting content, or simply a helpful reminder about the meetup: \n", + " - [Austin] Welcome Scott Aaronson to Texas: 13 August 2016 06:00PM - San Francisco Meetup: Group Debugging: 15 August 2016 06:15PM - Sydney Rationality Dojo—September 2016: 04 September 2016 04:00PM - Sydney Rationality Dojo—October 2016: 02 October 2016 04:00PM - Washington, D.C.: TED Talks: 14 August 2016 03:30PM Locations with regularly scheduled meetups:** Austin,** **Berlin, Boston, Brussels****, Buffalo, Canberra, Columbus, Denver,**** Kraków**,** ****London,**** ****Madison WI**,** ****Melbourne, Moscow,**** ****New Hampshire, ****New York, Philadelphia,**** Research Triangle NC,**** San Francisco Bay Area, ****Seattle, Sydney,**** Tel Aviv,**** ****Toronto, Vienna******, ****Washington DC****, and **West Los Angeles**.\n", + " - Title: New LW Meetup: Munich, Author: FrankAdamek, Date published: 2016-08-12, URL: https://www.lesswrong.com/posts/A9wkYLKoses5Q4Z3Q/new-lw-meetup-munich\n", + "\n", + "There’s also a 24⁄7 online study hall for coworking LWers and a Slack channel for daily discussion and online meetups on Sunday night US time. If you’d like to talk with other LW-ers face to face, and there is no meetup in your area, consider starting your own meetup; it’s easy (more resources here). Check one out, stretch your rationality skills, build community, and have fun! In addition to the handy sidebar of upcoming meetups, a meetup overview is posted on the front page every Friday. These are an attempt to collect information on all the meetups happening in upcoming weeks. The best way to get your meetup featured is still to use the Add New Meetup feature, but you’ll also have the benefit of having your meetup mentioned in a weekly overview. These overview posts are moved to the discussion section when the new post goes up. Please note that for your meetup to appear in the weekly meetups feature, you need to post your meetup *before *the Friday before your meetup! If you missed the deadline and wish to have your meetup featured, you can reach me on gmail at frank dot c dot adamek. L\n", + " - Title: New LW Meetup: Munich, Author: FrankAdamek, Date published: 2016-08-12, URL: https://www.lesswrong.com/posts/A9wkYLKoses5Q4Z3Q/new-lw-meetup-munich\n", + "\n", + "Whether or not there’s currently a meetup in your area, you can **sign up** to be notified automatically of any future meetups. And if you’re not interested in notifications you can still enter your approximate location, which will let meetup-starting heroes know that there’s an interested LW population in their city!\n", + "\n", + "If your meetup has a mailing list that you’d like mentioned here, or has become regular and isn’t listed as such, let me know!\n", + "\n", + "Want to help out the common good? If one of the meetups listed as regular has become inactive, let me know so we can present more accurate information to newcomers.\n", + "\n", + " - Title: New LW Meetup: Munich, Author: FrankAdamek, Date published: 2016-08-12, URL: https://www.lesswrong.com/posts/A9wkYLKoses5Q4Z3Q/new-lw-meetup-munich\n", + "\n", + "This recent discussion post by SarahC got me thinking about how one can rationally manage research. It seems like software might be useful here, but I don’t know how exactly the software should work. I’m intrigued by mind mapping software, but it’s possible that all that structure is unnecessary and you could do quite well with less. For instance, I’m considering trying a system of timestamped notes which are managed by a tagging system. If the tagging was done thoroughly enough, you could filter through all the posts sharing a cluster of tags and fairly easily get access to every idea you’ve recorded on a certain topic.\n", + "\n", + "The only problem is, I think I’d want even more specialized software than that. I’d want to integrate my notes with some form of bibliography management, and at least a to-do list. And I can imagine more, for example perhaps there could be a \"sticky note\" capability where I could pin up and move around things that I either want to remember or that will help me with my research, like an inspirational quote if I’m not feeling motivated to do research, or the Litany of Tarski or some other rationality technique if I really need to remember to use it.\n", + "\n", + "I’m not sure if these ideas are all sound, but a basic requirement for the software would be to document the structure of your research so that it can be analyzed for effectiveness.\n", + " - Title: How do you organize your research?, Author: nhamann, Date published: 2010-09-30, URL: https://www.lesswrong.com/posts/sa46t9uXaev2ZYZ9u/how-do-you-organize-your-research\n", + "\n", + "I know there are some people on Less Wrong who do research, so I suppose I should defer to the experts here: how do you organize your research? What methodologies and tools do you use? Why?\n", + "\n", + " - Title: How do you organize your research?, Author: nhamann, Date published: 2010-09-30, URL: https://www.lesswrong.com/posts/sa46t9uXaev2ZYZ9u/how-do-you-organize-your-research\n", + "\n", + "Contents\n", + " - What’s new at MIRI\n", + " - Fundraising goals\n", + " - Strategic background\n", + "**Update 2017-12-27:** We’ve blown past our 3rd and final target, and reached the matching cap of $300,000 for the $2 million Matching Challenge! Thanks so much to everyone who supported us!\n", + "\n", + "All donations made before 23:59 PST on Dec 31st will continue to be counted towards our fundraiser total. The fundraiser total includes projected matching funds from the Challenge.\n", + "\n", + "\n", + "MIRI’s **2017 fundraiser** is live through the end of December! Our progress so far (updated live):\n", + "\n", + "\n", + "\n", + "\n", + "Donate Now\n", + "\n", + "\n", + "MIRI is a research nonprofit based in Berkeley, California with a mission of ensuring that smarter-than-human AI technology has a positive impact on the world. You can learn more about our work at \"Why AI Safety?\" or via MIRI Executive Director Nate Soares’ Google talk on AI alignment.\n", + "\n", + "In 2015, we discussed our interest in potentially branching out to explore multiple research programs simultaneously once we could support a larger team. Following recent changes to our overall picture of the strategic landscape, we’re now moving ahead on that goal and starting to explore new research directions while also continuing to push on our agent foundations agenda. For more on our new views, see \"There’s No Fire Alarm for Artificial General Intelligence\" and our 2017 strategic update. We plan to expand on our relevant strategic thinking more in the coming weeks.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + "Our expanded research focus means that our research team can potentially grow big, and grow fast. Our current goal is to hire around ten new research staff over the next two years, mostly software engineers. If we succeed, our point estimate is that **our 2018 budget will be $2.8M** and **our 2019 budget will be $3.5M**, up from roughly $1.9M in 2017.1\n", + "\n", + "We’ve set our fundraiser targets by estimating how quickly we could grow while maintaining a 1.5-year runway, on the simplifying assumption that about 1⁄3 of the donations we receive between now and the beginning of 2019 will come during our current fundraiser.2\n", + "\n", + "Hitting **Target 1** ($625k) then lets us act on our growth plans in 2018 (but not in 2019); **Target 2** ($850k) lets us act on our full two-year growth plan; and in the case where our hiring goes better than expected, **Target 3** ($1.25M) would allow us to add new members to our team about twice as quickly, or pay higher salaries for new research staff as needed.\n", + "\n", + "We discuss more details below, both in terms of our current organizational activities and how we see our work fitting into the larger strategy space.\n", + "\n", + " What’s new at MIRI | Fundraising goals | Strategic background \n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + "# What’s new at MIRI\n", + "\n", + "New developments this year have included:\n", + "\n", + "\n", + " - The release of Eliezer Yudkowsky’s *Inadequate Equilibria: Where and How Civilizations Get Stuck*, a book on systemic failure, outperformance, and epistemology.\n", + "\n", + "\n", + " - New introductory material on decision theory: \"Functional Decision Theory,\" \"Cheating Death in Damascus,\" and \"Decisions Are For Making Bad Outcomes Inconsistent.\"\n", + "\n", + "\n", + " - Extremely generous new support for our research in the form of a one-time $1.01 million donation from a cryptocurrency investor and a three-year $3.75 million grant from the Open Philanthropy Project.3\n", + "\n", + "Thanks in part to this major support, we’re currently in a position to scale up the research team quickly if we can find suitable hires. We intend to explore a variety of new research avenues going forward, including making a stronger push to experiment and explore some ideas in implementation.4 This means that we’re currently interested in hiring exceptional software engineers, particularly ones with machine learning experience.\n", + "\n", + "The two primary things we’re looking for in software engineers are programming ability and value alignment. Since we’re a nonprofit, it’s also worth noting explicitly that we’re generally happy to pay excellent research team applicants with the relevant skills whatever salary they would need to work at MIRI. If you think you’d like to work with us, **apply here**!\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + "In that vein, I’m pleased to announce that we’ve made our first round of hires for our engineer positions, including:\n", + "\n", + "**Jesse Liptrap**, who previously worked on the Knowledge Graph at Google for four years, and as a bioinformatician at UC Berkeley. Jesse holds a PhD in mathematics from UC Santa Barbara, where he studied category-theoretic underpinnings of topological quantum computing.\n", + "\n", + "**Nick Tarleton**, former lead architect at the search startup Quixey. He previously studied computer science and decision science at Carnegie Mellon University, and Nick worked with us at the first iteration of our summer fellows program, studying consequences of proposed AI goal systems.\n", + "\n", + "On the whole, our initial hiring efforts have gone quite well, and I’ve been very impressed with the high caliber of our hires and of our pool of candidates.\n", + "\n", + "On the research side, our recent work has focused heavily on open problems in decision theory, and on other questions related to naturalized agency. Scott Garrabrant divides our recent work on the agent foundations agenda into four categories, tackling different AI alignment subproblems:\n", + "\n", + "**Decision theory** — Traditional models of decision-making assume a sharp Cartesian boundary between agents and their environment. In a naturalized setting in which agents are embedded in their environment, however, traditional approaches break down, forcing us to formalize concepts like \"counterfactuals\" that can be left implicit in AIXI-like frameworks.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + "Recent focus areas:\n", + "\n", + "\n", + " - As Rob noted in April, \"a common thread in our recent work is that we’re using probability and topological fixed points in settings where we used to use provability. This means working with (and improving) logical inductors and reflective oracles.\" Examples of applications of logical induction to decision theory include logical inductor evidential decision theory (\"Prediction Based Robust Cooperation,\" \"Two Major Obstacles for Logical Inductor Decision Theory\") and asymptotic decision theory (\"An Approach to Logically Updateless Decisions,\" \"Where Does ADT Go Wrong?\"). \n", + "\n", + " - Unpacking the notion of *updatelessness* into pieces that we can better understand, e.g., in \"Conditioning on Conditionals,\" \"Logical Updatelessness as a Robust Delegation Problem,\" \"The Happy Dance Problem.\"\n", + "\n", + " - The relationship between decision theories that rely on Bayesian conditionalization on the one hand (e.g., evidential decision theory and Wei Dai’s updateless decision theory), and ones that rely on counterfactuals on the other (e.g., causal decision theory, timeless decision theory, and the version of functional decision theory discussed in Yudkowsky and Soares (2017)): \"Smoking Lesion Steelman,\" \"Comparing LICDT and LIEDT.\" \n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + " - Lines of research relating to correlated equilibria, such as \"A Correlated Analogue of Reflective Oracles\" and \"Smoking Lesion Steelman II.\"\n", + "\n", + " - The Converse Lawvere Problem (1, 2, 3): \"Does there exist a topological space *X* (in some convenient category of topological spaces) such that there exists a continuous surjection from *X* to the space [0,1]*X* (of continuous functions from *X* to [0,1])?\"\n", + "\n", + " - Multi-agent coordination problems, often using the \"Cooperative Oracles\" framework.\n", + "\n", + "**Naturalized world-models** — Similar issues arise for formalizing how systems model the world in the absence of a sharp agent/​environment boundary. Traditional models leave implicit aspects of \"good reasoning\" such as causal and multi-level world-modeling, reasoning under deductive limitations, and agents modeling themselves.\n", + "\n", + "Recent focus areas:\n", + "\n", + "\n", + " - Kakutani’s fixed-point theorem and reflective oracles: \"Hyperreal Brouwer.\"\n", + "\n", + " - Transparency and merging of opinions in logical inductors.\n", + "\n", + " - *Ontology merging*, a possible approach to reasoning about ontological crises and transparency.\n", + "\n", + " - Attempting to devise a variant of logical induction that is \"Bayesian\" in the sense that its belief states can be readily understood as conditionalized prior probability distributions.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + "**Subagent avoidance** — A key reason that agent/​environment boundaries are unhelpful for thinking about AGI is that a given AGI system may consist of many different subprocesses optimizing many different goals or subgoals. The boundary between different \"agents\" may be ill-defined, and a given optimization process is likely to construct subprocesses that pursue many different goals. Addressing this risk requires limiting the ways in which new optimization subprocesses arise in the system.\n", + "\n", + "Recent focus areas:\n", + "\n", + "\n", + " - Benign induction: \"Maximally Efficient Agents Will Probably Have an Anti-Daemon Immune System.\" \n", + "\n", + " - Work related to KWIK learning: \"Some Problems with Making Induction Benign, and Approaches to Them\" and \"How Likely Is A Random AGI To Be Honest?\"\n", + "\n", + "**Robust delegation** — In cases where it’s desirable to delegate to another agent (e.g. an AI system or a successor), it’s critical that the agent be well-aligned and trusted to perform specified tasks. The value learning problem and most of the AAMLS agenda fall in this category. Recent focus areas:\n", + "\n", + "\n", + " - Goodhart’s Curse, \"the combination of the Optimizer’s Curse and Goodhart’s Law\" stating that \"a powerful agent neutrally optimizing a proxy measure *U* that we hoped to align with true values *V*, will implicitly seek out upward divergences of *U* from *V*\": \"The Three Levels of Goodhart’s Curse.\"\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + " - Corrigibility: \"Corrigibility Thoughts,\" \"All the Indifference Designs.\"\n", + "\n", + " - Value learning and inverse reinforcement learning: \"Incorrigibility in the CIRL Framework,\" \"Reward Learning Summary.\"\n", + "\n", + " - The reward hacking problem: \"Stable Pointers to Value: An Agent Embedded In Its Own Utility Function.\"\n", + "\n", + "Additionally, we ran several research workshops, including one focused on Paul Christiano’s research agenda.\n", + "\n", + "# Fundraising goals\n", + "\n", + "To a first approximation, we view our ability to make productive use of additional dollars in the near future as linear in research personnel additions. We don’t expect to run out of additional top-priority work we can assign to highly motivated and skilled researchers and engineers. This represents an important shift from our past budget and team size goals.5\n", + "\n", + "Growing our team as much as we hope to is by no means an easy hiring problem, but it’s made significantly easier by the fact that we’re now looking for top software engineers who can help implement experiments we want to run, and not just productive pure researchers who can work with a high degree of independence. (In whom we are, of course, still very interested!) We therefore think we can expand relatively quickly over the next two years (productively!), funds allowing.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + "In our mainline growth scenario, our reserves plus next year’s $1.25M installment of the Open Philanthropy Project’s 3-year grant would leave us with around 9 months of runway going into 2019. However, we have substantial uncertainty about exactly how quickly we’ll be able to hire additional researchers and engineers, and therefore about our 2018–2019 budgets.\n", + "\n", + "Our 2018 budget breakdown in the mainline success case looks roughly like this:\n", + "\n", + "2018 Budget Estimate (Mainline Growth)\n", + "\n", + "\n", + "\n", + "To determine our fundraising targets this year, we estimated the support levels (above the Open Philanthropy Project’s support) that would make us reasonably confident that we can maintain a 1.5-year runway going into 2019 in different growth scenarios, assuming that our 2017 fundraiser looks similar to next year’s fundraiser and that our off-fundraiser donor support looks similar to our on-fundraiser support:\n", + "\n", + "**Basic target — $625,000.** At this funding level, we’ll be in a good position to pursue our mainline hiring goal in 2018, although we will likely need to halt or slow our growth in 2019.\n", + "\n", + "**Mainline-growth target — $850,000.** At this level, we’ll be on track to fully fund our planned expansion over the next few years, allowing us to roughly double the number of research staff over the course of 2018 and 2019.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + "**Rapid-growth target — $1,250,000.** At this funding level, we will be on track to maintain a 1.5-year runway even if our hiring proceeds a fair amount faster than our mainline prediction. We’ll also have greater freedom to pay higher salaries to top-tier candidates as needed.\n", + "\n", + "Beyond these growth targets: if we saw an order-of-magnitude increase in MIRI’s funding in the near future, we have several ways we believe we can significantly accelerate our recruitment efforts to grow the team faster. These include competitively paid trial periods and increased hiring outreach across venues and communities where we expect to find high-caliber candidates. Funding increases beyond the point where we could usefully use the money to hire faster would likely cause us to spin off new initiatives to address the problem of AI x-risk from other angles; we wouldn’t expect them to go to MIRI’s current programs.\n", + "\n", + "On the whole, we’re in a very good position to continue expanding, and we’re enormously grateful for the generous support we’ve already received this year. Relative to our present size, MIRI’s reserves are much more solid than they have been in the past, putting us in a strong position going into 2018.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + "Given our longer runway, this may be a better year than usual for long-time MIRI supporters to consider supporting other projects that have been waiting in the wings. That said, we don’t personally know of marginal places to put additional dollars that we currently view as higher-value than MIRI, and we do expect our fundraiser performance to affect our growth over the next two years, particularly if we succeed in growing the MIRI team as fast as we’re hoping to.\n", + "\n", + "Donate Now\n", + "\n", + "# Strategic background\n", + "\n", + "Taking a step back from our immediate organizational plans: how does MIRI see the work we’re doing as tying into positive long-term, large-scale outcomes?\n", + "\n", + "A lot of our thinking on these issues hasn’t yet been written up in any detail, and many of the issues involved are topics of active discussion among people working on existential risk from AGI. In very broad terms, however, our approach to global risk mitigation is to think in terms of desired outcomes, and to ask: \"What is the likeliest way that the outcome in question might occur?\" We then repeat this process until we backchain to interventions that actors can take today.\n", + "\n", + "Ignoring a large number of subtleties, our view of the world’s strategic situation currently breaks down as follows:\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", " \n", + "1. **Long-run good outcomes**. Ultimately, we want humanity to figure out the best possible long-run future and enact that kind of future, factoring in good outcomes for all sentient beings. However, there is currently very little we can say with confidence about what desirable long-term outcomes look like, or how best to achieve them; and if someone rushes to lock in a particular conception of \"the best possible long-run future,\" they’re likely to make catastrophic mistakes both in how they envision that goal and in how they implement it. In order to avoid making critical decisions in haste and locking in flawed conclusions, humanity needs: \n", + "⬆ \n", + "2. A **stable period** during which relevant actors can accumulate whatever capabilities and knowledge are required to reach robustly good conclusions about long-run outcomes. This might involve decisionmakers developing better judgment, insight, and reasoning skills in the future, solving the full alignment problem for fully autonomous AGI systems, and so on. Given the difficulty of the task, we expect a successful stable period to require: \n", + "⬆ 3. A preceding **end to the acute risk period**. If AGI carries a significant chance of causing an existential catastrophe over the next few decades, this forces a response under time pressure; but if actors attempt to make irreversible decisions about the long-term future under strong time pressure, we expect the result to be catastrophically bad.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", + "\n", + "Conditioning on good outcomes, we therefore expect a two-step process where addressing acute existential risks takes temporal priority.\n", + "\n", + "To end the acute risk period, we expect it to be necessary for actors to make use of:\n", "\n", "\n", - "On chip local clock\n", - "In year 2013\n", - "In year 2014\n", - "In year 2015\n", + "⬆\n", "\n", "\n", - "Projection in 2003 ITRS\n", - "22.9 GHzTable 4c\n", - "Only odd years reported in this edition\n", - "33.4 GHzTable 4d\n", + "4. A **risk-mitigating technology**. On our current view of the technological landscape, there are a number of plausible future technologies that could be leveraged to end the acute risk period.\n", + "\n", + "We believe that the likeliest way to achieve a technology in this category sufficiently soon is through:\n", "\n", "\n", - "Projection in 2005 ITRS\n", + "⬆\n", "\n", - "28.4 GHzTable 4d\n", + "5. **AGI-empowered technological development** carried out by task-directed AGI systems. Depending on early AGI systems’ level of capital-intensiveness, on whether AGI is a late-paradigm or early-paradigm invention, and on a number of other factors, AGI might be developed by anything from a small Silicon Valley startup to a large-scale multinational collaboration. Regardless, we expect AGI to be developed before any other (meta)technology that can be employed to end the acute risk period, and if early AGI systems can be used safely at all, then we expect it to be possible for an AI-empowered project to safely automate a reasonably small set of concrete science and engineering tasks that are sufficient for ending the risk period. This requires:\n", + "⬆\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", "\n", "\n", + "6. **Construction of minimal aligned AGI**. We specify \"minimal\" because we consider success much more likely if developers attempt to build systems with the bare minimum of capabilities for ending the acute risk period. We expect AGI alignment to be highly difficult, and we expect additional capabilities to add substantially to this difficulty.\n", "\n", - "Projection in 2007 ITRS\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", - "\n", - "7.91 GHzTable 4c\n", - "\n", - "\n", - "\n", - "2014 reality\n", - "\n", - "4.0 GHz\n", - "Pricewatch.com\n", + "If an aligned system of this kind were developed, we would expect two factors to be responsible:\n", "\n", "\n", "\n", "\n", - "The most conspicuous issue is that the 2003 and 2005 editions overstated clock rate by about 7x. ITRS accommodated to multicore in 2007 with a new scaling model that we see in retrospect overstates reality by only 2x. Footnote 1 in the 2007 ITRS describes the change. The footnote ends with the following sentence: “This is to reflect recent on-chip frequency slowing trends and anticipated speed-power design tradeoffs to manage a maximum 200 watts/chip affordable power management tradeoff.”\n", - "If you believe ITRS is “industry,” industry had been telling customers to expect the benefits of Moore’s Law through rising clock rate. In my view, customers took the lead in saying power per chip should be less than 200 watts even if it meant a more difficult to use parallel programming model. Several years after the multicore became popular, industry changed its projection so customers were to expect the benefits of progress through rising computations per dollar rather than speed. This, of course, led to the rise of battery operated smart phones and tablets with power limits much lower than 200 watts.\n", - "By the way, I have not heard the phrase “Moorean trend” before. It seems to capture the idea of progress in computing without being tied to particular technical property. Why don’t you trademark it; it gets zero Google hits.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", "\n", - "Luke: Are you willing to make some forecasts about the next ~15 years in computing? I’d be curious to hear your point estimate, or even better your 70% confidence interval, for any of the following: FLOPS per US dollar in top-end supercomputing in 2030.\n", - "Average kT per active logic gate in top-end supercomputing in 2030.\n", - "Some particular measure of progress on reversible computing, in 2030?\n", - "World’s total FLOPS capacity in 2030. (See here.) Or really, anything specific about computing you’d like to forecast for 2030. Erik: The FLOPS per dollar question will be most interesting, so I’ll leave it for last. kT/logic op: I see a plateau around 10,000 kT, and will discuss what might come beyond the plateau in the next paragraph. . My guess of 10,000 kT includes interconnect wire, which is significant because today 75-90% of energy is attributable to interconnect wire. Today, we see around 50,000 kT. A reduction in supply voltage to . 3v should be good for 10x improvement, but there are other issues. This estimate should be valid in 10 years, but the question asked about 15 years. I would not be surprised that we see a new approach in the interval 2025-2030 (mentioned below). It will be difficult to predict specifically, but the five-year interval is short and the improvement rate seems to be insensitive to details. So, say there is a 5x additional improvement by 2030. Cumulative by 2030: 2,000 kT/logic op, including interconnect wire. However, this will be really disappointing.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", "\n", - "People will expect 10 doublings due to Moore’s Law in the 15-year interval, for an expected improvement of 1024x; I’m predicting 25x. Reversible computing: I think reversible computing (as strictly defined) will be demonstrated in a few years and principally impact society’s thought processes. The demonstration would be computation at less than 1 kT/logic op, where theory says those levels are unachievable unless reversible computing principles are used. I do not expect reversible computing to be widely used by 2030. The projection of 2,000 kT/logic op in 2030 represents a balance of manufacturing costs and energy costs. By 2030, reversible computing could be employed in some applications where power is very expensive, such as spacecraft or implantable medical devices. However, a demonstration of reversible computing could have an important impact on societal thinking. Popular thinking sees some ideas as unlimited for planning purposes and endows those ideas with attention and investment. This applied to California real estate prices (until 2008) and Moore’s Law (until a few years ago). Claims that “Moore’s Law is Ending” are moving computation into a second class of ideas that popular thinking sees as limited, like the future growth potential of railroads. A reversible computing demonstration would move computing back to the first category and thus make more attention and capital available. However, reversible computing is part of a continuum.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", + "⬆\n", "\n", - "I see a good possibility that adiabatic methods could become the new method mentioned above for the 2020-2025 time range. World’s Total FLOPS capacity, 2030. I looked over the document by Naik you cited. I don’t feel qualified to judge his result. However, I will stand by my ratio of 50,000 kT to 2,000 kT = 25. So my answer is to multiply Naik’s result by 25. I do not imagine that the cumulative power consumption of computers will rise substantially, particularly with Green initiatives. FLOPS per dollar: This answer will be all over the place. Let’s break down by application class: (A) Some applications are CPU-bound, meaning their performance will track changes in kT per logic op. I have given my guess of 25x improvement (which is a lot less then the 1024x that Moore’s Law would have delivered). (B) Other applications are memory bound, meaning their performance will track (a) memory subsystem performance, where advances partially overlap with advances due to Moore’s Law and (b) architecture changes that can reduce the amount of data movement. It is a lot easier to make a computer for (A) than (B); for a given cost, a computer will outperform type A applications by an order of magnitude or more on FLOPS compared to type B. A top-end supercomputer supports both A and B, but the balance between A and B may be the profound question of the era. The balance has been heavily weighted in favor of A (through reliance on LINPACK as the benchmark).\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", + "7a. A **technological edge in AGI by a strategically adequate project**. By \"strategically adequate\" we mean a project with strong opsec, research closure, trustworthy command, a commitment to the common good, security mindset, requisite resource levels, and heavy prioritization of alignment work. A project like this needs to have a large enough lead to be able to afford to spend a substantial amount of time on safety measures, as discussed at FLI’s Asilomar conference.\n", "\n", - "However, we do not currently have a particularly aggressive Exascale program in the US. Instead, we have a lot of discussion about the memory subsystem’s low energy efficiency. You can make a fairly compelling case that progress in top-end supercomputing will be held up until the computers can become better balanced. (For reference the TOP 2 supercomputer is ORNL Titan with 17.5 Petaflops Linpack for $97 million; a ratio of 181 MFLOPS/$. The TOP 1 supercomputer does not seem to be a good cost reference. ) If architecture stays fixed until 2030, I’ll guess 25x improvement. That would be 4.5 GFLOPS/$. Memory subsystems are made out of the same transistor technology as logic, perhaps plus a growing fraction of optics. If transistors become more effective by 25x, this could benefit both FLOPS and the memory subsystem. Use of 3D may boost performance (due to shorter wires), but this will be offset by efficiency loss due to difficulty exploiting greater parallelism. Call the latter factors a wash. Architecture is the wildcard. There are architectures known that are vastly more efficient than the von Neumann machine, such as systolic arrays, Processor-In-Memory (PIM), Field Programmable Gate Arrays (FPGAs) and even GPUs. These architectures get a performance boost by organizing themselves to put calculations closer to where data is stored, requiring less time and energy to complete a task. Unfortunately, these architectures succeed at the expense of generality.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", "\n", - "If a vendor boosts performance through too much specialization, they run the risk of being disqualified as an example of a “top-end supercomputer”. The Holy Grail would be a software approach that would make general software run on some specialized hardware (like a compiler that would run general C code on a GPU – at the full performance of the GPU). However, I will predict that architecture improvements will contribute an additional 4x by 2030, for a cumulative improvement factor of 100x. That will be 18 GFLOPS/$. This is still 10x short of the 1024x expected for 15 years. However, I think Artificial General Intelligence (AGI) may fare well due to specialization. Synaptic activity is the dominant function that enables living creatures to think, but it is quite different from the floating point in a supercomputer. A synapse performs local, slow, computations based on analog stimuli and analog learned behavior. In contrast, the floating point in a supercomputer operates blazingly fast on data fetched from a distant memory and computes an answer with 64-bit precision. Speed reduces energy efficiency, and the supercomputer doesn’t even learn. Since a von Neumann computer is Turing complete, it will be capable of executing an AGI coded in software. However, the efficiency may be low. Executing an AGI could be optimized by new or specialized technology and advance faster than the rate of Moore’s Law, like Bitcoin mining.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", + "⬆\n", "\n", - "I am going to project that an AGI demonstration at scale will require a non-conventional, but not unimaginable computer.\n", "\n", - "The computer could be specialized CMOS, like a GPU with special data types and data layout.\n", + "7b. A strong **white-boxed system understanding** on the part of the strategically adequate project during late AGI development. By this we mean that developers go into building AGI systems with a good understanding of how their systems decompose and solve particular cognitive problems, of the kinds of problems different parts of the system are working on, and of how all of the parts of the system interact.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", "\n", - "Alternatively, the computer could employ new physical devices, such as a neuromorphic architecture with a non-transistor device (e.\n", + "On our current understanding of the alignment problem, developers need to be able to give a reasonable account of how all of the AGI-grade computation in their system is being allocated, similar to how secure software systems are built to allow security professionals to give a simple accounting of why the system has no unforeseen vulnerabilities. See \"Security Mindset and Ordinary Paranoia\" for more details.\n", "\n", - "g. memristor).\n", + "Developers must be able to explicitly state and check all of the basic assumptions required for their account of the system’s alignment and effectiveness to hold. Additionally, they need to design and modify AGI systems only in ways that preserve understandability — that is, only allow system modifications that preserve developers’ ability to generate full accounts of what cognitive problems any given slice of the system is solving, and why the interaction of all of the system’s parts is both safe and effective.\n", "\n", - "All said, AGI might see 1000x or more improvement.\n", + "Our view is that this kind of system understandability will in turn require:\n", "\n", - "In other words, AGI enthusiasts might be able to plan on 181 GFLOPS/$ by 2030.\n", "\n", - "However, they would be classed as AI machines rather than top-end supercomputers.\n", + "⬆\n", "\n", - "Luke: Thanks, Erik!\n", - "Note added in review: Landauer proposed a lower limit on “on the order of kT” only for “irreversible” computations. As far as I know, the phrase “Landauer’s Limit” was created later by other people. In my experience, the phrase “Landauer’s Limit” if often applied as a general limit.The post Erik DeBenedictis on supercomputing appeared first on Machine Intelligence Research Institute.\n", - " - Title: Erik DeBenedictis on supercomputing, Author: Luke Muehlhauser, Date published: Thu, 03 Apr 2014, URL: https://intelligence.org/2014/04/03/erik-debenedictis/\n", "\n", - "Speaker 00:03 Hello and Welcome to Session 146 in the AC T comm reading group. Tonight we'll be discussing the second part of categories article, likelihood of discontinuous progress around the development of artificial general intelligence. Catch a grace is probably the author of this piece, and the lead scientist at the AI impacts projects. We're discussing the second half of the article wherein categories describes her summary of the arguments for this discontinuous progress, and why she is not convinced that the likelihood is particularly large. And as before, I've tried to add some kind of answers both to the presentation of the arguments and to categorises counter arguments, mostly from the book super intelligence. But before I get into that, I would like to once again say thank you to catch a grace. Getting a high quality, critical thoughts and scepticism about these arguments is really important. They're my counter arguments this time are a bit more disjointed than last time, because my true objection for why I believe there is a large likelihood of discontinuous progress was for the section intelligence explosion that we discussed last time. So for this reason, in the other. In the other arguments, I'm kind of trying to push it towards this talk about Bostrom six, two superpowers that we had last week. I also tried to include some other counter arguments where I see them, but I'm less convinced about them, but I wanted to add them anyway.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "8. **Steering toward alignment-conducive AGI approaches**. Leading AGI researchers and developers need to deliberately direct research efforts toward ensuring that the earliest AGI designs are relatively easy to understand and align.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", "\n", - "So the first section is called train versus test. And, and it also includes searching actually, so it should maybe be search versus train versus test. And test in this case is deploying and executing. So it might take, just like with a metaphor for martial arts, you spent much more time training that you do fighting. And this is the basic idea, compute can be used for three things, both searching the space of all possible software programmes that can be written and see which ones are AGI eyes. And once you have something that is perhaps an AGI, then you can try to train it. And once you have trained it, then you can try to run it. And there is a hierarchy here in that the first takes much longer than the second, and the second takes much longer than the third. This is just some kind of discontinuity in impact, in that if you are at a if you have computer processes, processes that are good enough for step two, then they can probably run the AGI very, very fast. And that might create some kind of more impact than you'd otherwise think. catchphrases counter argument for this is that the AGI will still while it's an improvement before it will be used, is it still something that replaces slightly worse software. And that means the discontinuity we'll be seeing is not a discontinuity on a high level capability, but only a small step ahead of what was previously the state of the art.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "We expect this to be a critical step, as we do not expect most approaches to AGI to be alignable after the fact without long, multi-year delays. \n", + "We plan to say more in the future about the criteria for strategically adequate projects in **7a**. We do not believe that any project meeting all of these conditions currently exists, though we see various ways that projects could reach this threshold. The above breakdown only discusses what we view as the \"mainline\" success scenario.6 If we condition on good long-run outcomes, the most plausible explanation we can come up with cites a strategically adequate AI-empowered project ending the acute risk period, and appeals to the fact that those future AGI developers maintained a strong understanding of their system’s problem-solving work over the course of development, made use of advance knowledge about which AGI approaches conduce to that kind of understanding, and filtered on those approaches. For that reason, MIRI does research to intervene on **8** from various angles, such as by examining holes and anomalies in the field’s current understanding of real-world reasoning and decision-making. We hope to thereby reduce our own confusion about alignment-conducive AGI approaches and ultimately help make it feasible for developers to construct adequate \"safety-stories\" in an alignment setting.\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", "\n", - "One of the things that I noticed here was the the the argument the kind of steel men first talked about when we discover AGI and catch a grace here is talking about the gradual step towards AGI. And it's something that will, at first be only a tiny, tiny, while continues bit better than what we have before. So So I guess this hints at one of the key philosophical differences there is, will a is there a missing key to general intelligence that we haven't discovered? Yes, I think most scientists would say yes, that there is still some mysteries locked in the brain. But of course, it is possible even when there is that AGI will be gradual, but it's also possible that it will be much more abrupt. Now for the the actual counter-arguments. A week one that I just realised was that of course, if we have training time, that is very long on the order of years, then this year, we will have some kind of we'll have better hardware to do most law and this kind of thing. And that might be another reason why the AGI would be potentially faster. That's not really my true objection. My true reaction is the only example we have of General Lynch origins are humans, training a human takes roughly 18 years give or take, and 18 years is actually requires quite a lot of compute to go through 18 years.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "As we improve our understanding of the alignment problem, our aim is to share new insights and techniques with leading or up-and-coming developer groups, who we’re generally on good terms with.\n", "\n", - "So if we imagine that we have two, two systems, one with an IQ of 100, and the other with an IQ of 101, so an extremely continuous, a tiny, tiny bit of difference, but then the, if we can train them in 18 years, we can probably just as easily execute them for 18 years. So if you imagine 18 people thinking for a year, that's the same as one person thinking for 18 years, if you have the people with an IQ of 101 101, then what would be the difference? My intuition is that since it goes, since the the slightly better, algorithms will be around for so much longer time, there could be a strong discontinuous difference between the two. Another issue is that this train time of 18 years, is something that evolution has forced on humans for because we need to be able to procreate and, and this kind of thing. So humans need to learn quickly. But actually, it's not, it's not obvious that 18 years at all, is the optimal in any, for any particular reason. It might be that to reach a higher level of AGI, you need an even greater difference between training time and execution time. But it's of course also possible it can go the other way, it might be that training time is not appreciably longer than testing time, we don't know basically, because we do not have AGI yet. Now, if we include searching for computer programmes, then we imagine some kind of three step model where we start by searching for programmes, and then the ones we have found, we train them.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "A number of the points above require further explanation and motivation, and we’ll be providing more details on our view of the strategic landscape in the near future.\n", "\n", - "And once we've trained, we test them, then this is a loop that requires a lot more, a lot more computing power. This, the difference between train and test is something that we that is brought up from time to time, but searching programmes. That's not something people talk a lot about. Because searching for programmes is in general something that computer scientists only do if they have no other choice, because it is normally extraordinary, computationally expensive. In people off more often talk about the search for programmes in the context of what like the universal prior and what programmes we can find there. And a lot of people's think that that is like some of the most unsafe AIS, but let's assume that we can't build AGI in any other ways. So we will find it by searching. So we're in the far future now. Now, how much more space does it take to find computer programmes compared to running them? Well, probably almost certainly enough that just if you need to search, then train a particular computer programme needs to be basically trivial. So the factors that I intuitively, intuitively think about here is that searching compared to training, then searching takes a billion billions of times more time. And then of course, training and test might also be a building of a factor. So this means that the factors here are not like the difference between 118 people thinking for one year, but like billions of people thinking for billions of years.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "Further questions are always welcome at contact@intelligence.org, regarding our current organizational activities and plans as well as the long-term role we hope to play in giving AGI developers an easier and clearer shot at making the first AGI systems robust and safe. For more details on our fundraiser, including corporate matching, see our **Donate** page.\n", "\n", - "And of course here that would magnify the differences to a truly discontinuous extent. The next argument is about starting Hi. Where can you create is as an example, using the Wright brothers. Wright Flyer three, I believe this picture is I that was actually in the beginning, I made a no categories promise that there will be a separate treatment of the case where inventions go from zero to one. And I said that that wasn't in the first part, but actually it is in the second part here. So this argument, is that the first item in a trend, that's a discontinuity in the opposite sense that the amount of examples go from zero to one. And so this is claimed by category to not be very interesting. The first, there's a metric on how many AGI systems do we have? And that's not really interesting. I would actually kind of think for the radical point of view and all these kind of things. It's actually really, really interesting, but that's it not interesting, according to Bostrom, six superpowers. Now the problem here is that the development of AGI that could cause some metrics to go from zero to some high number according to social impact. And when we think about Boston's superpowers, then we actually don't care about going from the level siru. To human level, that's unlikely to have great impact. What's going to happen, great impact is going from human level to above. Now, how often does it happen on the outside view, that new technologies have this kind of high social impact?\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", "\n", - "The example that categories uses is aeroplane size, which Wright Flyer was 40 feet. So that's kind of large, and there was never a time where the best aircraft were atomic size, they started at 40 feet and only grow larger by that. But aeroplane size, that's not really a very socially impactful metric. And, again, here, as before, we are playing this kind of this game of reference class tennis where categories are throwing a lot of examples are saying, maybe the development of AGI would be like the development. Well, not really like the development of aircraft, but our planes size. Our plane size is a really, really odd metric. I've never heard about that before. And so I'm wondering if this could be some kind of strawman. I think the same argument might be make with how long time the first right fly could fly, right? There was never a time where there was aircraft. But But aircrafts were not able to fly for one minute, they did start out being able to fly, you know, appreciably length and flying for one minute, that's not that bad, really. Now, my, my true objection about this is that, in the example with Wright Flyer, what they what the brothers right, actually did, that was truly groundbreaking, was to make a lot of groundwork, they built models in wind tunnels, and did a huge amount of work on actually building this these aircrafts.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "1 Note that this $1.9M is significantly below the $2.1–2.5M we predicted for the year in April. Personnel costs are MIRI’s most significant expense, and higher research staff turnover in 2017 meant that we had fewer net additions to the team this year than we’d budgeted for. We went under budget by a relatively small margin in 2016, spending $1.73M versus a predicted $1.83M.\n", "\n", - "And that was actually where their true their true contribution was, in figuring out how to build them, and not so much in actually flying them. That was more like, Okay, that was maybe also it's kind of a contribution, but the big thing were all the preliminary work they did, but they never went public with that. And the public probably would not have care about this, the public cares about inventions. At the time, when it is above the alternative, they don't care about that you have made a better wind tunnel and figuring out how to do that in a good way. They become public in at the level where they are above the alternative. There are some cases where people don't make them public at this particular case, the military have a number of things where they have kept secrets, there are companies that are kept secrets, they probably keep them secret in the hope of a maybe discontinuous, decisive strategic advantage, or something else. So my point here is that it's not a coincidence that we often see inventions made public at precisely the first level where they are above the the alternative. That's because the incentives to make things public, make that the right choice. Of course, this is a really imperfect model I haven't thought so much about it's possible to get quite a few to think of quite a few exceptions to this trend. Now catchphrases, counter arguments go in a somewhat different direction.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "Our 2018–2019 budget estimates are highly uncertain, with most of the uncertainty coming from substantial uncertainty about how quickly we’ll be able to take on new research staff. ↩\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", "\n", - "She claims that looking back, new technologies usually don't use these kind of discontinuities. The examples you she's using are very different from the ones I used in the in the previous presentation. We're talking about the agricultural revolution, the scientific revolution, the Industrial Revolution, and catcher Grace are thinking about some very different reference classes. She's claiming here, I'm quoting at length, it is hard to imagine AGI suddenly processing some new trade to a degree that will revolutionise the economy without this producing discontinuous change in previously measured things like ability to turn resources into financial profit. And I've come up with what I think is a reasonable counter example to this. If we imagine an AGI gradually improving. First it's capable of doing intellectual work according to an IQ of 80 and then It improves. So it's capable of doing intellectual work comparable to an IQ of 120. That would not give humanity as such any. Any new skills, we can't do anything that we couldn't do before, because there are people with an ad with a higher IQ. But this would, of course, still revolutionise the economy. Categories also says there's no trend where important technologies start high in general. And I probably agree with this. I don't really know how you would go about determining this in a robust way. But I think she's probably right. So I'm not gonna argue against that.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "2 This is roughly in line with our experience in previous years, when excluding expected grants and large surprise one-time donations. We’ve accounted for the former in our targets but not the latter, since we think it unwise to bank on unpredictable windfalls.\n", "\n", - "Now, there are also the arguments for standing Hi, that we're using the inside view. And this, of course, is a picture from Bostrom spoke super intelligence showing the kinetics of an intelligence explosion. And so what are the reasons we should expect AGI looking to exceed this kind of base rate that we saw on the outside view, the base rate probability of standing high? Well, argument one is that it's a very novel technology. So we're more likely to see progress on metrics that have seen no progress before. And since I don't really care about these, most of these metrics, I care about six metrics, which are buzzwords, six superpowers, and of course, there have been dramatic progress on this. So I think I reject this argument out of the box, then there is the argument that full AGI would be unusually impactful. And that implies that minimum AGI would also be impactful. And here, if I can just go back to past Trump's graph here, where there's a takeoff starting here, where the AGI begins to contribute to its own development, and then it ends up with some kind of strong super intelligence here. And that means that the minimum functional AGI here is the first AGI that's capable of contributing to its own development that makes it impactful by definition on the superpower called intelligence amplification.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "Note that in previous years, we’ve set targets based on maintaining a 1-year runway. Given the increase in our size, I now think that a 1.5-year runway is more appropriate. ↩\n", "\n", - "So what we back basically will see is that anything below this level is something that the the project will work on, but not really publicised or used for anything in particular. So it is basically for all intents and purposes, not AGI but an AGI being built. At least, that's one way of looking at AGI and that's probably not how catch a grace looks at AGI. But unfortunately, she doesn't really comment on this kinetics of an intelligence explosion from mushrooms, books, so I can't really see what I'm arguing against here. The third argument on the inside view is that it's possible that AGI is for some reason not not amenable to coming in functional, but low quality. And the reason why I believe this is that in this the time until take off, while the AGI is being built, we will see the AGI that it might be functional, but as it has too low quality to actually do anything, then most likely the researchers will not will not release it or do much other things with this project, except just improving it until it gets to the level where it's able to meaningfully contribute to its own development. Categories this counter argument against again go in a somewhat different direction. The original argument that if fully developed AGI is unusual, impactful, this suggests the minimal functional AGI also will be unusually impactful. And catchy grace gives us as an example, humans as an general intelligence that is imperfect but functional. And that is indeed true.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "3 Including the $1.01 million donation and the first $1.25 million from the Open Philanthropy Project, we have so far raised around $3.16 million this year, overshooting the $3 million goal we set earlier this year! ↩\n", "\n", - "And that is of course precisely what what is going on in classrooms. Description of this takeoff, where humans and the minimal AGI collaborate to improve the AGI. So, one way to see this in short ways that if if it's not high if the AGI is not impactful, then it's not really AGI then it has a start. Categories also use this as a positive example of showing how it's possible to have functional AGI but not have an intelligence explosion. And that is usually explained by hardware limitation to rise. But actually I think hardware is only rather small Part of it, the true reason why humans haven't done and intelligence explosion is that we can't do brain surgery on ourselves. But with software, you can actually edit software in a much easier way. The next document is on Alpha zero, which is claimed to be some kind of discontinuity, at least within games, and perhaps also other metrics. So that means that technologists like Alpha zero probably also have some kind of greater pro probability of producing discontinuities, categories, countless, that the matrix that officer was also on, were not really interesting. And they she's not really examining other AI progress, particularly closely forward discontinuities. She doesn't believe that this is particularly relevant. I actually feel that it might be relevant. Go and chess are examples of matrix where people actually had put quite a lot of thought into how to build chess, chess games and go and go algorithms.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "4 We emphasize that, as always, \"experiment\" means \"most things tried don’t work.\" We’d like to avoid setting expectations of immediate success for this exploratory push. ↩\n", "\n", - "So I think calling it reasonably optimised before alphazero can be justified, then whether it is worthwhile to look at other AI progress? Well, I actually think I don't, this is not my true objection. But I think she's going a bit too quickly over this. The article is called likelihood of discontinuous AI progress. And then saying we don't care about a how many discontinuities there are in AI progress. That seems like an obviously relevant thing that is excluded here. But that's truly my objection. So in general, what I feel Elsa serum is proof of is that it is apparently possible in AI to have these kinds of discontinuities, probably because AI is not truly optimised the same way that humans optimise, trying to predict stock prices or things like that. There's just not enough people who are working in DeepMind, or comparable places. Then the next argument is about uneven skills, where the argument is that if we want to replace a human doing skill, A, B, C, D, and E, then we need to have some kind of ability on all of them. And if once the human level AI is above, on all, in particular skill, see here, then on the other skills like D, we might have something that is very strongly superhuman. I'm not quite sure about this argument to any particular degree, I feel that we're using the word general intelligence. And this general intelligence probably can be used reasonably interchangeable, not perfectly, but reasonably interchangeable for all these skills.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "5 Our previous goal was to slowly ramp up to the $3–4 million level and then hold steady with around 13–17 research staff. We now expect to be able to reach (and surpass) that level much more quickly. ↩\n", "\n", - "So that's more like my intuition about how this work. But I don't have very strong arguments for this catchphrases, counter arguments, this is that this is actually something we see whenever one kind of process replace and other. And since categories do leave that discontinuities are rare in practice, then from that follows this dynamic doesn't really tend to produce great discontinuities. My answer, one of the weaker as I add to this is that the six superpowers that we're looking at, they are extremely socially impactful. So for this reason, we are actually also interested in in even minor discontinuities in the categories further argues that humans are not generally replaced in one go, but step by step. And the problem is that this is how it's been so far. And this is how it's been for machines and also for narrow AI. But AGI is truly different. On the inside view in the we are replacing the core of what makes humans able to perform a lot of these skills. So for this reason, there is a much greater reason to expect powerful AGI to replace basically all humans in one go. Then there is a lot of thresholds. The last two arguments are about thresholds, both payoff thresholds and human competition thresholds.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "6 There are other paths to good outcomes that we view as lower-probability, but still sufficiently high-probability that the global community should allocate marginal resources to their pursuit. ↩\n", "\n", - "I feel it might be better to take them at least a bit together in the argument is given the first if we assume we have some kind of linear progress in AGI then that might still cause some very discontinuous change in economic value and ability to obtain it the session To take advantage, we have even more examples here, AGI might be comparable to a boat, in that building better boats might not help you until you have a boat that's actually capable of reaching the next island. So if the universe is made up of Ireland's in this way, then there might be some thresholds that will cause this continuous change. And it is also possible that this threshold might not be in nature. As the example with the ions, it might be that going from not competitive to humans to slightly better than humans would yield a qualitatively different outcome. I feel here that this is actually from two different sections. And I feel they're mixed somewhat together in categories, this description. If we take samples from the pay, not from the human competition thresholds, then its economic value and ability to take over the world. And this seems like something that relates very strongly to competing with humans, we want to have economic value than the way and AGI has economic value is by being better than humans or comparable to humans, and taking over the world. While humans have the world right now. So it's taking over the world from humans. Catch a grace has yet another example of what AGI might be.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + " - Title: MIRI’s 2017 Fundraiser, Author: Malo, Date published: 2017-12-07, URL: https://www.lesswrong.com/posts/shy6XPhbrh9az6NG6/miri-s-2017-fundraiser\n", "\n", - "And she claims that AGI might be comparable to the value of a projectile fired from a gun according to its speed. This is another rather strange example. So the assumption is, if the market correctly predicts what is the value according to the speed of the projectile, then the market will assign more resources to building guns that are worse on all other metrics, but have a higher speed and a more expensive so that the this extra resources will go to having a gun that reached precisely this threshold in speed. And in the equilibrium there's there'll be no discontinuity around the time when a gun with this particular speed hits the market. I feel this is probably true. But it hinges on that the market is correctly able to predict how will what will be the impact of artificial general intelligence. And I think this is unlikely there has been cases where it has been true, we have seen things like the invention of the light bulb, where there was an enormously long period where people tried to build light bulbs, and then they put some that were extremely expensive and only could only be used and then gradually was better and better. So arguably, the value created by the light bulb was introduced in a completely discontinuous way. There are many other examples of markets failing to predict technologies. One of the strongest example I found was in was actually not directly related to technology.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "## Discussion article for the meetup : MelbLW Social Meetup: Apr 2016\n", "\n", - "But the Cuban Missile Crisis, which potentially would have had an enormous influence on on the market, where the market basically didn't react at all to the world coming to the brink of nuclear annihilation. I feel this is a I mean, David might disagree here, but there is a limit to what the market can predict. In particular, I feel that when a technology will be if there is ever a technology that will allow an ex to obtain decide strategic advantage. My strong intuition is that the market will not price it correctly in advance. Because basically what price would it be it would be infinitely high. Then there's a bit more about the human competition thresholds finally, and here I'm basically not summarising catchphrases article, but more going out with a with my own interpretation of the arguments. And that is the human competition threshold seem to be particularly important in six domains, intelligence amplification, hacking Bussum, six superpowers, basically, those are particularly relevant in how strongest the AI compared to humans in these activities. Categories answer is that human skills vary a lot. So it's actually not much of a threshold because there is a huge difference in how productive are humans and how skilled are they at social manipulation, etc. That might be true, but some of them are not done by People who score very Allah, I've tried to sort them here.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", "\n", - "So I feel like, for instance, intelligence amplification in a, in an intelligence explosion scenario, this is something that will not be done by most people, but be done by a modestly small team at DeepMind or something like that, which is a tiny, tiny percentage of the human variation.\n", "\n", - "For hacking also, this is where the skip the top skill matter lot, and the lowest schools don't really matter.\n", + "**WHEN:**\n", + "15 April 2016 06:30:00PM (+1000)\n", "\n", - "So we might imagine, to agents to AIS version 0.9 and 0.1.\n", "\n", - "Again, we're the first hacker it's better than 99.9% of all humans.\n", "\n", - "So that means that there are 7 million humans that are better than the than the as a hacker, and the different AI, which is slightly better, it is better than 100% of humans.\n", + "**WHERE:**\n", + "Bull and Bear Tavern, 347 Flinders Lane, Melbourne VIC 3000\n", "\n", - "And in this case, I would expect the first hacker to have basically no impact, because there isn't there's hardly 7 million people trying to stop hackers.\n", "\n", - "Whereas if there is hacker there is that is better than the humans, then then the humans that are trying to stop the hacker, then that will would likely have an enormous impact because it might be able to make an attack that precisely zero of our current systems can can withstand.\n", "\n", - "So for this reason, I believe that this kind of threshold might be really, really important.\n", "\n", - "But that is all for today.\n", + "The LessWrong Melbourne Social Meetup is on again, this Friday!\n", "\n", - "Thank you and see you next week.\n", - " - Title: 146. Discontinuous Progress 2-by AI Safety Reading Group-video_id j8GnMy-Bckk-da\n", + "Facebook event page: https://​​www.facebook.com/​​events/​​921062154679530/​​\n", + "\n", + "Social Meetups occur on the third Friday of each month. They are casual get-togethers where we chat over a few drinks and a meal. Sometimes we play games as well.\n", + "\n", + "**Where?** This month we are back at the Bull & Bear Tavern in Flinders Lane, near Queen St. There will be a comedy event on in the Bistro area at the back, but we will be downstairs in the Tavern area (with the pool tables).\n", + "\n", + "**When?** Friday 15th April, starting from 6:30 pm. Start time is very loose so feel free to come whenever suits you. We usually stay at the main venue until about 11pm.\n", + "\n", + "**Dinner?** The Bull & Bear serve decent pub food and we usually get a plate of wedges or two to share as well. Those who are still keen typically go for a late dinner after the event, and lately we’ve been frequenting Father’s Office which offers late night a-la-carte menu at half price (!).\n", + "\n", + "**Contact?** Any questions or issues, just call or SMS Richard on 0421231789.\n", + " - Title: Meetup : MelbLW Social Meetup: Apr 2016, Author: Maelin, Date published: 2016-04-10, URL: https://www.lesswrong.com/posts/w79LCn5HkCgSB7Si4/meetup-melblw-social-meetup-apr-2016\n", + "\n", + "Hope to see you there!\n", + "\n", + "\n", + "## Discussion article for the meetup : MelbLW Social Meetup: Apr 2016\n", + "\n", + " - Title: Meetup : MelbLW Social Meetup: Apr 2016, Author: Maelin, Date published: 2016-04-10, URL: https://www.lesswrong.com/posts/w79LCn5HkCgSB7Si4/meetup-melblw-social-meetup-apr-2016\n", + "\n", + "Details and central discussion are here on Reddit, but I want to also reach the part of the LW community that would be interested in weighing in but doesn’t regularly use /​r/​HPMOR or /​r/​rational. Keeping the discussion centralized on Reddit would be preferred, but if you’d really rather comment /​ PM here then go for it.\n", + "\n", + " - Title: I’m planning a rationalist technique-demonstrating sequel to HPMOR. I know about the Virtue of Silence. Are there any skills I should leave out?, Author: b_sen, Date published: 2016-03-01, URL: https://www.lesswrong.com/posts/Lrqx7XCkqQ9kXk9wQ/i-m-planning-a-rationalist-technique-demonstrating-sequel-to\n", + "\n", + "I’ve seen various contenders for the title of simplest abstract game that’s interesting enough that a professional community could reasonably play it full time. While Go probably has the best ratio of interest to complexity, Checkers and Dots and Boxes might be simpler while remaining sufficiently interesting. [1] But is Checkers actually simpler than Go? If so, how much? How would we decide this?\n", + "\n", + "Initially you might approach this by writing out rules. There’s an elegant set for Go and I wrote some for Checkers, but English is a very flexible language. Perhaps my rules are underspecified? Perhaps they’re overly verbose? It’s hard to say.\n", + "\n", + "A more objective test is to write a computer program that implements the rules. It needs to determine whether moves are valid, and identify a winner. The shorter the computer program, the simpler the rules of the game. This only gives you an upper bound on the complexity, because someone could come along and write a shorter one, but in general we expect that shorter programs imply shorter possible programs.\n", + "\n", + "To investigate this, I wrote ones for each of the three games. I wrote them quickly, and they’re kind of terse, but they represent the rules as efficiently as I could figure out. The one for Go is based off Tromp’s definition of the rules while the other two implement the rules as they are in my head. This probably gives an advantage to Go because those rules had a lot of care go into them, but I’m not sure how much of one.\n", + " - Title: Does Checkers have simpler rules than Go?, Author: jefftk, Date published: 2013-08-13, URL: https://www.lesswrong.com/posts/buf87jaKck83EkNzH/does-checkers-have-simpler-rules-than-go\n", + "\n", + "The programs as written have some excess information, such as comments, vaguely friendly error messages, whitespace, and meaningful variable names. I took a jscompiler-like pass over them to remove as much of this as possible, and making them nearly unreadable in the process. Then I ran them through a lossless compressor, gzip, and computed their sizes:\n", + "\n", + "\n", + " - Checkers: 648 bytes \n", + "\n", + " - Dots and Boxes: 505 bytes \n", + "\n", + " - Go: 596 bytes \n", + "\n", + "(The programs are on github. If you have suggestions for simplifying them further, send me a pull request.)\n", + "\n", + " [1] Go is the most interesting of the three, and has stood up to centuries of analysis and play, but Dots and Boxes is surprisingly complex (pdf) and there used to be professional Checkers players. (I’m having a remarkably hard time determining if there are still Checkers professionals.)\n", + "\n", + "*I also posted this on my blog.*\n", + "\n", + " - Title: Does Checkers have simpler rules than Go?, Author: jefftk, Date published: 2013-08-13, URL: https://www.lesswrong.com/posts/buf87jaKck83EkNzH/does-checkers-have-simpler-rules-than-go\n", + "\n", + "Is there any conjunction fallacy research which addresses the alternative hypothesis that the observed results are mainly due to social dynamics?\n", + "\n", + "Most people spend most of their time thinking in terms of gaining or losing social status, not in terms of reason. They care more about their place in social status hierarchies than about logic. They have strategies for dealing with communication that have more to do with getting along with people than with getting questions technically right. They look for the social meaning in communications. E.g. people normally try to give – and expect to receive – useful, relevant, reasonable info that is safe to make socially normal assumptions about.\n", + "\n", + "Suppose you knew Linda in college. A decade later, you run into another college friend, John, who still knows Linda. You ask what she’s up to. John says Linda is a bank teller, doesn’t give additional info, and changes the subject. You take this to mean that there isn’t more positive info. You and John both see activism positively and know that activism was one of the main ways Linda stood out. This conversation suggests to you that she stopped doing activism. Omitting info isn’t neutral in real world conversations. People mentally model the people they speak with and consider why the person said and omitted things.\n", + " - Title: [Question] Can Social Dynamics Explain Conjunction Fallacy Experimental Results?, Author: curi, Date published: 2020-08-05, URL: https://www.lesswrong.com/posts/tKcdTsMFkYjnFEQJo/can-social-dynamics-explain-conjunction-fallacy-experimental\n", + "\n", + "In Bayesian terms, you got two pieces of info from John’s statement. Roughly: 1) Linda is a bank teller. 2) John thinks that Linda being a bank teller is key info to provide and chose not to provide other info. That second piece of info can affect people’s answers in psychology research.\n", + "\n", + "So, is there any research which rules out social dynamics explanations for conjunction fallacy experimental results?\n", + "\n", + " - Title: [Question] Can Social Dynamics Explain Conjunction Fallacy Experimental Results?, Author: curi, Date published: 2020-08-05, URL: https://www.lesswrong.com/posts/tKcdTsMFkYjnFEQJo/can-social-dynamics-explain-conjunction-fallacy-experimental\n", + "\n", + "Contents\n", + " - Motivation\n", + " - Experiment description\n", + " - Experiment details\n", + " - Prize structure\n", + " - Desk research prizes\n", + "(*This question comes from a discussion with Carl Shulman*.)\n", + "In this post I describe an experiment that I’d like to see run. I’m posting a $1,000 - $10,000 prize for a convincing implementation of these experiments. I also post a number of smaller prizes for relevant desk research or important corrections to this request.\n", + "## Motivation\n", + "In order to understand the dynamics of the singularity, I’d like to understand how easy it is to improve algorithms and software.\n", + "We can learn something about this from looking at chess engines. It’s not the most relevant domain to future AI, but it’s one with an unusually long history and unusually clear (and consistent) performance metrics.\n", + "In order to quantify the quality of a chess engine, we can fix a level of play and ask \"How much compute is needed for the engine to play at that level?\"\n", + "One complication in evaluating the rate of progress is that it depends on what level of play we use for evaluation. In particular, newer algorithms are generally designed to play at a much higher level than older algorithms. So if we quantify the compute needed to reach modern levels of play, we will capture both absolute improvements and also \"adaptation\" to the new higher amounts of compute.\n", + "So we’d like to attribute progress in chess engines to three factors:\n", + " - Title: How much chess engine progress is about adapting to bigger computers?, Author: paulfchristiano, Date published: 2021-07-07, URL: https://www.lesswrong.com/posts/H6L7fuEN9qXDanQ6W/how-much-chess-engine-progress-is-about-adapting-to-bigger\n", + "\n", + " - Better software. - Bigger computers. - Software that is better-adapted to new, bigger computers.\n", + "Understanding the size of factor #1 is important for extrapolating progress given massive R&D investments in software. While it is easy to separate factors #1 and #2 from publicly available information, it is not easy to evaluate factor #3.\n", + "## Experiment description\n", + "Pick two (or more) software engines from very different times. They should both be roughly state of the art, running on \"typical\" machines from the era (i.e. the machines for which R&D is mostly targeted).\n", + "We then carry out two matches: - Run the old engine on its \"native\" hardware (the \"old hardware\"). Then evaluate: how little compute does the new engine need in order to beat the old engine? - Run the new engine on its \"native\" hardware (the \"new hardware\"). Then evaluate: how much compute does the old engine need in order to beat the new engine? With some effort, we can estimate a quantitative ratio of \"ops needed\" for each of these experiments. For example, we may find that the new engine is able to beat the old engine using only 1% of the \"old hardware\". Whereas we may find that the old engine would require 10,000x the \"new hardware\" in order to compete with the new engine. The first experiment tells us about the absolute improvements in chess engines on the task for which the old engine was optimized. (This understates the rate of software progress to the extent that people stopped working on this task.\n", + " - Title: How much chess engine progress is about adapting to bigger computers?, Author: paulfchristiano, Date published: 2021-07-07, URL: https://www.lesswrong.com/posts/H6L7fuEN9qXDanQ6W/how-much-chess-engine-progress-is-about-adapting-to-bigger\n", + "\n", + ") The second experiment gives us the combination of absolute improvements + adaptation to new hardware.\n", + "\n", + "Typical measures of \"rate of software progress\" will be somewhere in between, and are sensitive to the hardware on which the evaluation is carried out.\n", + "\n", + "I believe that understanding these two numbers would give us a significantly clearer picture of what’s really going on with software progress in chess engines.\n", + "\n", + "## Experiment details Here’s some guesses about how to run this experiment well.\n", + "\n", + "I don’t know much about computer chess, so you may be able to make a better proposal.\n", + "\n", + " - **Old engine, old hardware**: my default proposal is the version of Fritz that won the 1995 world computer chess championship, using the same amount of hardware (and time controls) as in that championship. This algorithm seems like a particularly reasonable \"best effort\" at making full use of available computing resources. I don’t want to compare an engine running on a very expensive old machine to an engine running on a cheap modern machine. You may have to be opportunistic about what kind of thing you can actually run.\n", + "\n", + " - **New engine, new hardware**: my default proposal is the version of Stockfish that won TCEC Season 20, on the same hardware+time used in that competition.\n", + " - Title: How much chess engine progress is about adapting to bigger computers?, Author: paulfchristiano, Date published: 2021-07-07, URL: https://www.lesswrong.com/posts/H6L7fuEN9qXDanQ6W/how-much-chess-engine-progress-is-about-adapting-to-bigger\n", + "\n", + " - **Running a new engine on old hardware**: We should use whatever modern engine works best with teensy computers. It’s not important it be the same as the modern engine on modern hardware. We’d prefer if there was a dedicated team continuing to work on this problem, but absent that we want to use the best thing that exists.\n", + "\n", + " - **Memory use**. When running on the old hardware we need to match the memory use of the old machine. I’m not sure how to handle scaling of memory. One possibility is to hold the ratio fixed and scale them both up/​down together, but something more realistic would be welcome.\n", + "\n", + " - **Matchup vs ELO**: I proposed experiments organized around 1:1 contests. I think there are lots of ways that could go wrong. It would be really good to at least sanity-check the results by comparing against a third reference engine. Quantification of this factor is welcome.\n", + "\n", + " - **More engines, more hardware**: you could do the same experiment with additional software engines, and could evaluate at more levels of compute. I think you get a lot of the benefits from the first few measurements, but more data helps and it might be a lot cheaper (especially if you need more engines to get good ELO estimates anyway).\n", + " - Title: How much chess engine progress is about adapting to bigger computers?, Author: paulfchristiano, Date published: 2021-07-07, URL: https://www.lesswrong.com/posts/H6L7fuEN9qXDanQ6W/how-much-chess-engine-progress-is-about-adapting-to-bigger\n", + "\n", + " - **Even older engines**. I’m interested in results even older than Fritz, but the further back we go the more uncertainty I have about how to actually do the comparison.\n", + "\n", + " - **Endgame tables, opening book, learned heuristics**: some of the knowledge produced by chess engines was produced *using* large computers (by playing or observing very large numbers of games, brute-forcing endgames, and so on). We’d prefer exclude these factors if they use more compute than would have been affordable for the old engine. If we can’t, we at least want to quantify their influence. Including these factors could significantly overstate the returns of human R&D; it’s an important thing to know about, but for the purpose of forecasting the impacts of AI we really want to separate it out. This may be a major constraint for considering new engines.\n", + "\n", + " - **What counts as an \"operation\"?** I don’t think that making \"hardware\" comparisons between new and old computers will be straightforward. I think it’s easier if we restrict attention to consumer microprocessors, and I’m hoping that there’s only a little bit of uncertainty (e.g. a factor of 2). I think you can do the experiments before figuring this out, and then try to clarify relevant issues by seeing how fast the computers can run simple relevant backgrounds. (The \"old hardware\" run should probably be on just one processor.)\n", + " - Title: How much chess engine progress is about adapting to bigger computers?, Author: paulfchristiano, Date published: 2021-07-07, URL: https://www.lesswrong.com/posts/H6L7fuEN9qXDanQ6W/how-much-chess-engine-progress-is-about-adapting-to-bigger\n", + "\n", + " - **Timing out**. It may be prohibitively expensive to give the old engine enough compute to beat the new engine. I think you should do a little bit of that work, to try to figure out the basic picture (how expensive it would be, rough bounds on the numbers, some very noisy estimates from playing just a few games). We can figure out where to go from there. - **Different time controls**. I’m expecting time controls to be comparable between the old and new competitions. If old competitions ran on much longer time controls, I’d prefer scale them down to something comparable (to try to better match what would have been realistic to experiment with during R&D /​ profitable for non-competition purposes). And similarly if new competitions are longer. - **Ponder**. Thinking during the opponent’s turn could change the answer a *tiny* amount (up to a factor of 2), but it really doesn’t seem worth dealing with, even if some of the engines are optimized to use it. - **Inefficiencies from running on new computers**. You might have to mess things up a lot to run old engines on new computers (or to deal with weird amounts of memory or so on). Ideally it would be possible to abstract out some of the details of \"how long it actually takes\" to talk about what the computation or memory costs ought to be if implemented without overhead. We’ll have to see how that goes.\n", + " - Title: How much chess engine progress is about adapting to bigger computers?, Author: paulfchristiano, Date published: 2021-07-07, URL: https://www.lesswrong.com/posts/H6L7fuEN9qXDanQ6W/how-much-chess-engine-progress-is-about-adapting-to-bigger\n", + "\n", + "## Prize structure **I’m not spending long evaluating any of this and my decisions are going to be completely unaccountable. Dealing with me may be a waste of time. Your only recourse will be to make an angry comment on this post. ** **I know that all makes it less appealing to participate. But please have it in mind before spending any time on this project, and consider yourself warned! ** I’m planning to give away at least $1,000 sometime in the next 3 months if anyone runs any plausible version of this experiment, or even points me to public information from which it’s possible to back out a similar number. I’m fine if that means I have to give $1,000 to a random LW comment. I’d give away $10,000 if someone ran a clean version of the experiment, I found it convincing, and they were transparent about their methods. Before giving away the full prize I’d likely have a waiting period where I offered a separate prize for people to post replications or caveats or so on. I’m generally expecting/​hoping to wrap this up over the next couple of months, but will adjust based on responses. If multiple people do experiments I’ll make some arbitrary call about how to allocate prize money. Timing will matter a bit but it’s a secondary consideration to quality of experiment. Earlier submissions can get paid based if they helped pave the way for later submissions. If you are planning to spend time on this I encourage you to make a comment.\n", + " - Title: How much chess engine progress is about adapting to bigger computers?, Author: paulfchristiano, Date published: 2021-07-07, URL: https://www.lesswrong.com/posts/H6L7fuEN9qXDanQ6W/how-much-chess-engine-progress-is-about-adapting-to-bigger\n", + "\n", + "I’m very happy to provide thoughts on a proposed experiment, e.g. to tell you what size of prize I’d expect to give it or what concerns I’d have about a proposal.\n", + "\n", + "**None of this is binding**.\n", + "\n", + "If receiving a prize would be weird or problematic for some reason, I’m still interested in the results and you can opt to receive a comparable quantity of gratitude instead of $.\n", + "\n", + " - Plausible (or better yet convincing) estimates of the total investment in chess engine R&D over time.\n", + "\n", + " - Good analysis of the relative importance of hardware/​software using public information (must at least improve over the analysis here). Or pointers to other similar experiments that have already been run.\n", + "\n", + " - Any consideration that I feel should significantly change the experimental setup, e.g. quantifying the importance of endgame tables, or noting a critical difference between old and new chess engines, or suggesting a reason that Fritz is a bad engine to compare to.\n", + "\n", + " - Any contributions that make it significantly easier to run this experiment (for example tracking down a usable implementation of an old chess engine).\n", + "\n", + " - Title: How much chess engine progress is about adapting to bigger computers?, Author: paulfchristiano, Date published: 2021-07-07, URL: https://www.lesswrong.com/posts/H6L7fuEN9qXDanQ6W/how-much-chess-engine-progress-is-about-adapting-to-bigger\n", + "\n", + "Lately I’ve come to think of human civilization as largely built on the backs of intelligence and virtue signaling. In other words, civilization depends very much on the positive side effects of (not necessarily conscious) intelligence and virtue signaling, as channeled by various institutions. As evolutionary psychologist Geoffrey Miller says, \"it’s all signaling all the way down.\"\n", + "\n", + "A question I’m trying to figure out now is, what determines the relative proportions of intelligence vs virtue signaling? (Miller argued that intelligence signaling can be considered a kind of virtue signaling, but that seems debatable to me, and in any case, for ease of discussion I’ll use \"virtue signaling\" to mean \"other kinds of virtue signaling besides intelligence signaling\".) It seems that if you get too much of one type of signaling versus the other, things can go horribly wrong (the link is to Gwern’s awesome review/​summary of a book about the Cultural Revolution). We’re seeing this more and more in Western societies, in places like journalism, academia, government, education, and even business. But what’s causing this?\n", + "\n", + "One theory is that Twitter with its character limit, and social media and shorter attention spans in general, have made it much easier to do virtue signaling relative to intelligence signaling. But this seems too simplistic and there has to be more to it, even if it is part of the explanation.\n", + " - Title: [Question] What determines the balance between intelligence signaling and virtue signaling?, Author: Wei_Dai, Date published: 2019-12-09, URL: https://www.lesswrong.com/posts/vA2Gd2PQjNk68ngFu/what-determines-the-balance-between-intelligence-signaling\n", + "\n", + "Another idea is that intelligence is valued more when a society feels threatened by an outside force, for which they need competent people to protect themselves from. US policy changes after Sputnik is a good example of this. This may also explain why intelligence signaling continues to dominate or at least is not dominated by virtue signaling in the rationalist and EA communities (i.e., we’re really worried about the threat from Unfriendly AI). Does anyone have other ideas, or have seen more systematic research into this question? Once we understand the above, here are some followup questions: Is the trend towards more virtue signaling at the expense of intelligence signaling likely to reverse itself? How bad can things get, realistically, if it doesn’t? Is there anything we can or should do about the problem? How can we at least protect our own communities from runaway virtue signaling? (The recent calls against appeals to consequences make more sense to me now, given this framing, but I still think they may err too much in the other direction.) PS, it was interesting to read this in Miller’s latest book Virtue Signaling: > Where does the term ‘virtue signaling’ come from? Some say it goes back to 2015, when British journalist/​author James Bartholomew wrote a brilliant piece for The Spectator called ‘The awful rise of ‘virtue signaling. ’’ Some say it goes back to the Rationalist blog ‘LessWrong,’ which was using the term at least as far back as 2013.\n", + " - Title: [Question] What determines the balance between intelligence signaling and virtue signaling?, Author: Wei_Dai, Date published: 2019-12-09, URL: https://www.lesswrong.com/posts/vA2Gd2PQjNk68ngFu/what-determines-the-balance-between-intelligence-signaling\n", + "\n", + "Even before that, many folks in the Rationalist and Effective Altruism subcultures were aware of how signaling theory explains a lot of ideological behavior, and how signaling can undermine the rationality of political discussion.\n", + "\n", + "\n", + "I didn’t know that \"virtue signaling\" was first coined (or at least used in writing) on LessWrong. Unfortunately, from a search, it doesn’t seem like there was substantial discussion around this term. Signaling in general was much discussed on LessWrong and OvercomingBias, but I find myself still updating towards it being more important than I had realized.\n", + "\n", + " - Title: [Question] What determines the balance between intelligence signaling and virtue signaling?, Author: Wei_Dai, Date published: 2019-12-09, URL: https://www.lesswrong.com/posts/vA2Gd2PQjNk68ngFu/what-determines-the-balance-between-intelligence-signaling\n", + "\n", + "Hi All,\n", + "I was wondering if anyone knows of a dataset with probabilistic predictions for a binary outcome that has published results for calibration loss, resolution, and refinement loss? We’ve developed some code and an accompanying web tool for generating calibration plots, ROC curves, and the aforementioned metrics and wanted to test it out on validated data before unleashing it formally. \n", + "\n", + " - Title: [Question] Test Data for Calibration Analysis, Author: behrang, Date published: 2019-11-21, URL: https://www.lesswrong.com/posts/Hxqx46joeaZHkDfEf/test-data-for-calibration-analysis\n", + "\n", + "Contents - Experiencing Evidence - Memory Collection - Imaginal Exposure - Experiencing Beliefs - The Acceptance Statement - Experiencing Felt Senses - Sitting With - Expressing - Experiencing Metaphors - Clean Language At this level, you’re simply sitting with the schema. Oftentimes, this alone can produce a dramatic shift through memory reconsolidation simply as a matter of re-evaluating the old schema when you get a look at it. I also include non-memory reconsolidation techniques in this category that work through extinction of the old association. # Experiencing Evidence ## Memory Collection Memory collection involves finding and allowing yourself to experience every time you can remember when you felt or acted through a particular belief schema. For each piece of evidence, you allow yourself to think and feel through the schema in that particular memory, then move on to the next one. By simply allowing ourselves to see all the evidence for a schema with our current perspective, we can often come to a more nuanced understanding of the belief. I first learned about this technique through PJ Eby,who calls it the Feeling Elimination Technique. The book, thinking things done, that it was included in, is no longer available as far as I’m aware. ## Imaginal Exposure Imaginal exposure is a process of taking a particular salient memory/​fear/​trauma etc, and simply running through the experience over and over.\n", + " - Title: Reconsolidation Through Experience, Author: Matt Goldenberg, Date published: 2019-11-13, URL: https://www.lesswrong.com/posts/QfgAgkahWtHfp2xGA/reconsolidation-through-experience\n", + "\n", + "This can often trigger memory reconsolidation, and, even if not, can change your schema through simple exhaustion induced extinction. **EMDR** is a version of imaginal exposure that adds the modality of moving your eyes back and forth as you run through the memory. My current best view of the evidence is that EMDR has no long term benefit over imaginal exposure, but the data is confusing. # Experiencing Beliefs ## The Acceptance Statement An acceptance statement is a tool from Coherence Therapy, that involves taking a belief, and the choices you’ve made about a belief and saying them out loud. For instance I could state out loud \"I should not express myself. If I do, I’ll be ostracized. I’ve decided it’s more important to not be ostracized than to be fully seen\". Saying this out loud is often enough to trigger the schema, and hearing out out loud is often enough to automatically trigger the questioning of the schema. You can enhance this process by (in your imagination or in real life), saying the statement outloud to someone who has been affected by the choices you’ve made. I could imagine saying to my partner: \"I should not express myself. If I do, you will ostracize me. I’ve decided it’s more important to not be ostracized than to be fully seen by you\". **Emotional Freedom Technique** is a process that combines saying acceptance statements out loud with tapping various parts of your body.\n", + " - Title: Reconsolidation Through Experience, Author: Matt Goldenberg, Date published: 2019-11-13, URL: https://www.lesswrong.com/posts/QfgAgkahWtHfp2xGA/reconsolidation-through-experience\n", + "\n", + "It’s unclear to me based on the evidence if the tapping adds anything outside of simply stating the beliefs.\n", + "\n", + "# Experiencing Felt Senses ## Sitting With A common recommendation is simply to spend some time sitting with your feelings.\n", + "\n", + "By simply sitting with the feeling, it gives you time to process it, allowing you to reconsolidate the schema with your new processed understanding.\n", + "\n", + "## Expressing Another way to experience felt senses is to express them.\n", + "\n", + "**Focusing** is one process that allows you to express your felt senses, as are common forms of expressive therapy like Art Therapy.\n", + "\n", + "# Experiencing Metaphors ## **Clean Language** Clean Language is a series of questions you can ask yourself to start exploring your own metaphors.\n", + "\n", + "Simply by exploring our metaphors, we can often get a new perspective and reconsolidate them.\n", + " - Title: Reconsolidation Through Experience, Author: Matt Goldenberg, Date published: 2019-11-13, URL: https://www.lesswrong.com/posts/QfgAgkahWtHfp2xGA/reconsolidation-through-experience\n", + "\n", + "Link post Contents - Highlights - Technical AI alignment - Iterated amplification - Forecasting - Miscellaneous (Alignment) - Other progress in AI - Exploration - Reinforcement learning - Applications Find all Alignment Newsletter resources here. In particular, you can sign up, or look through this spreadsheet of all summaries that have ever been in the newsletter. I’m always happy to hear feedback; you can send it to me by replying to this email. Audio version here (may not be up yet). ## **Highlights** Learning Latent Plans from Play *(Corey Lynch et al)* (summarized by Cody): This paper collects unsupervised data of humans playing with robotic control systems, and uses that data to thread a needle between two problems in learning. One problem is that per-task demonstration data is costly, especially as number of tasks grows; the other is that randomly sampled control actions will rarely stumble across complex motor tasks in ways that allow robots to learn. The authors argue that human play data is a good compromise because humans at play tend to explore different ways of manipulating objects in ways that give robots nuggets of useful information like \"how do I move this block inside a drawer\", which can be composed into more complicated and intentional tasks. The model works by learning to produce vectors that represent plans (or sequences of actions), and jointly learning to decode those vectors into action sequences.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "This architecture learns to generate plan vectors by using an autoencoder-like structure that uses KL divergence to align (1) a distribution of plan vectors predicted from the start and end state of a window of play data, and (2) a distribution of plan vectors predicted by looking back at all the actions taken in that window. Because we’re jointly learning to unroll the (2) lookback-summarized vector such that it matches the actions actually taken, we’ll ideally end up with a system that can take in a given plan vector and produce a sequence of actions to execute that plan. And, because we’re learning to predict a vector that aligns with actions successfully taken to get to an end state from a starting one, the model at test time should be able to produce a play vector corresponding to feasible actions that will get it from its current state to a goal state we’d like it to reach. The authors found that their Play-trained model was able to outperform single-task models on a range of manipulation tasks, even though those single-task models were trained with explicit demonstrations of the task. **Cody’s opinion:** I really liked this paper: it was creative in combining conceptual components from variational methods and imitation learning, and it was pragmatic in trying to address the problem of how to get viable human-demonstration data in a way that avoids having to get distinct datasets for a huge set of different discrete tasks.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "# **Technical AI alignment** ### **Iterated amplification** Aligning a toy model of optimization *(Paul Christiano)* (summarized by Rohin): Current ML capabilities are centered around **local search**: we get a gradient (or an approximation to one, as with evolutionary algorithms), and take a step in that direction to find a new model. Iterated amplification takes advantage of this fact: rather than a sequence of gradient steps on a fixed reward, we can do a sequence of amplification steps and distillation gradient steps. However, we can consider an even simpler model of ML capabilities: **function maximization**. Given a function from n-bit strings to real numbers, we model ML as allowing us to find the input n-bit string with the maximum output value, **in only O(n) time** (rather than the O(2^n) time that brute force search would take). If this were all we knew about ML capabilities, could we still design an aligned, competitive version of it? While this is not the actual problem we face, **due to its simplicity it is more amenable to theoretical analysis**, and so is worth thinking about. We could make an unaligned AI that maximizes some explicit reward using only 2 calls to Opt: first, use Opt to find a good world model M that can predict the dynamics and reward, and then use Opt to find a policy that does well when interacting with M. This is unaligned for all the usual reasons: most obviously, it will try to seize control of the reward channel.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "An aligned version does need to use Opt, since **that’s the only way of turning a naively-exponential search into a linear one**; without using Opt the resulting system won’t be competitive. We can’t just generalize iterated amplification to this case, since iterated amplification relies on a *sequence* of applications of ML capabilities: this would lead to an aligned AI that uses Opt many times, which will not be competitive since the unaligned AI only requires 2 calls to Opt. One possible approach is to design an AI with good incentives (in the same way that iterated amplification aims to approximate HCH (AN #34)) that \"knows everything that the unaligned AI knows\". However, it would also be useful to produce a proof of impossibility: this would tell us something about what a solution must look like in more complex settings. **Rohin’s opinion:** Amusingly, I liked this post primarily because comparing this setting to the typical setting for iterated amplification was useful for seeing the design choices and intuitions that motivated iterated amplification. a\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "(For example, we could ask how many people will invest in safety. ) Then, by aggregating across survey respondents, we can see whether or not the researchers beliefs about what others will do match the empirical distribution of what researchers are planning to do. This can help mitigate the effect where everyone thinks that everyone else will deal with a problem, and the effect where everyone tries to solve a problem because they all think no one else is planning to solve it. Critch has offered to provide suggestions on including this methodology in any upcoming surveys; see the post for details. **Rohin’s opinion:** This is a cool idea, and seems worth doing to me. I especially like that the survey would simply reveal problems by collecting two sources of information from people and checking their consistency with each other: there isn’t any particular argument being made; you are simply showing inconsistency in people’s own beliefs to them, if and only if such inconsistency exists. In practice, I’m sure there will be complications—for example, perhaps the set of researchers taking the survey is different from the set of \"others\" whose actions and beliefs they are predicting—but it still seems worth at least trying out. AI Forecasting Dictionary *(Jacob Lagerros and Ben Goldhaber)* (summarized by Rohin): One big challenge with forecasting the future is operationalizing key terms unambiguously, so that a question can be resolved when the future actually arrives.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "Since we’ll probably need to forecast many different questions, it’s crucial that we make it as easy as possible to create and answer well-operationalized questions. To that end, the authors have created and open-sourced an AI Forecasting Dictionary, which gives precise meanings for important terms, along with examples and non-examples to clarify further. AI Forecasting Resolution Council *(Jacob Lagerros and Ben Goldhaber)* (summarized by Rohin): Even if you operationalize forecasting questions well, often the outcome is determined primarily by factors other than the one you are interested in. For example, progress on a benchmark might be determined more by the number of researchers who try to beat the benchmark than by improvements in AI capabilities, even though you were trying to measure the latter. To deal with this problem, an AI Forecasting Resolution Council has been set up: now, forecasters can predict what the resolution council will say at some particular time in the future. This allows for questions that get at what we want: in the previous case, we could now forecast how the resolution council will answer the question \"would current methods be able to beat this benchmark\" in 2021. How to write good AI forecasting questions + Question Database *(Jacob Lagerros and Ben Goldhaber)* (summarized by Rohin): As discussed above, operationalization of forecasting questions is hard.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "This post collects some of the common failure modes, and introduces a database of 76 questions about AI progress that have detailed resolution criteria that will hopefully avoid any pitfalls of operationalization. ### **Miscellaneous (Alignment)** The strategy-stealing assumption *(Paul Christiano)* (summarized by Rohin): We often talk about aligning AIs in a way that is *competitive* with unaligned AIs. However, you might think that we need them to be *better*: after all, unaligned AIs only have to pursue one particular goal, whereas aligned AIs have to deal with the fact that we don’t yet know what we want. We might hope that regardless of what goal the unaligned AI has, any strategy it uses to achieve that goal can be turned into a strategy for acquiring *flexible* influence (i.e. influence useful for many goals). In that case, **as long as we control a majority of resources**, we can use any strategies that the unaligned AIs can use. For example, if we control 99% of the resources and unaligned AI controls 1%, then at the very least we can split up into 99 \"coalitions\" that each control 1% of resources and use the same strategy as the unaligned AI to acquire flexible influence, and this should lead to us obtaining 99% of the resources in expectation. In practice, we could do even better, e.g. by coordinating to shut down any unaligned AI systems.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "The premise that we can use the same strategy as the unaligned AI, despite the fact that we need *flexible* influence, is called the **strategy-stealing assumption**. Solving the alignment problem is critical to strategy-stealing—otherwise, unaligned AI would have an advantage at thinking that we could not steal and the strategy-stealing assumption would break down. This post discusses **ten other ways that the strategy-stealing assumption could fail**. For example, the unaligned AI could pursue a strategy that involves threatening to kill humans, and we might not be able to use a similar strategy in response because the unaligned AI might not be as fragile as we are. **Rohin’s opinion:** It does seem to me that if we’re in a situation where we have solved the alignment problem, we control 99% of resources, and we aren’t infighting amongst each other, we will likely continue to control at least 99% of the resources in the future. I’m a little confused about how we get to this situation though—the scenarios I usually worry about are the ones in which we fail to solve the alignment problem, but still deploy unaligned AIs, and in these scenarios I’d expect unaligned AIs to get the majority of the resources. I suppose in a multipolar setting with continuous takeoff, if we have mostly solved the alignment problem but still accidentally create unaligned AIs (or some malicious actors create them deliberately), then this setting where we control 99% of the resources could arise.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "# **Other progress in AI** ### **Exploration** Making Efficient Use of Demonstrations to Solve Hard Exploration Problems *(Caglar Gulcehre, Tom Le Paine et al)* (summarized by Cody): This paper combines ideas from existing techniques to construct an architecture (R2D3) capable of learning to solve hard exploration problems with a small number (N~100) of demonstrations. R2D3 has two primary architectural features: its use of a recurrent head to learn Q values, and its strategy of sampling trajectories from separate pools of agent and demonstrator experience, with sampling prioritized by highest-temporal-difference-error transitions within each pool. As the authors note, this approach is essentially an extension of an earlier paper, Deep Q-Learning from Demonstrations, to use a recurrent head rather than a feed-forward one, allowing it to be more effectively deployed on partial-information environments. The authors test on 8 different environments that require long sequences of task completion to receive any reward, and find that their approach is able to reach human level performance on four of the tasks, while their baseline comparisons essentially never succeed on any task. d\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "**Cody’s opinion:** For all that this paper’s technique is a fairly straightforward merging of existing techniques (separately-prioritized demonstration and agent pools, and the off-policy SotA R2D2), its results are surprisingly impressive: the tasks tested on require long and complex chains of correct actions that would be challenging for a non-imitation based system to discover, and high levels of environment stochasticity that make a pure imitation approach difficult. ### **Reinforcement learning** Emergent Tool Use from Multi-Agent Interaction *(Bowen Baker et al)* (summarized by Rohin): We have such a vast diversity of organisms and behaviors on Earth because of evolution: every time a new strategy evolved, it created new pressures and incentives for other organisms, leading to new behaviors. The multiagent competition led to an *autocurriculum*. This work harnesses this effect: they design a multiagent environment and task, and then use standard RL algorithms to learn several interesting behaviors. Their task is hide-and-seek, where the agents are able to move boxes, walls and ramps, and lock objects in place. The agents find *six* different strategies, each emerging from incentives created by the previous strategy: seekers chasing hiders, hiders building shelters, seekers using ramps to get into shelters, hiders locking ramps away from seekers, seekers surfing boxes to hiders, and hiders locking both boxes and ramps.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "The hope is that this can be used to learn general skills that can then be used for specific tasks. This makes it a form of unsupervised learning, with a similar goal as e.g. curiosity (AN #20). We might hope that multiagent autocurricula would do better than curiosity, because they automatically tend to use features that are important for control in the environment (such as ramps and boxes), while intrinsic motivation methods often end up focusing on features we wouldn’t think are particularly important. They empirically test this by designing five tasks in the environment and checking whether finetuning the agents from the multiagent autocurricula learns faster than direct training and finetuning curiosity-based agents. They find that the multiagent autocurricula agents do best, but only slightly. To explain this, they hypothesize that the learned skill representations are still highly entangled and so are hard to finetune, whereas learned feature representations transfer more easily. **Rohin’s opinion:** This is somewhat similar to AI-GAs (AN #63): both depend on *environment design*, which so far has been relatively neglected. However, AI-GAs are hoping to create *learning algorithms*, while multiagent autocurricula leads to *tool use*, at least in this case. Another point of similarity is that they both require vast amounts of compute, as discovering new strategies can take significant exploration.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "That said, it seems that we might be able to drastically decrease the amount of compute needed by solving the exploration problem using e.g. human play data or demonstrations (discussed in two different papers above).\n", + "\n", + "More speculatively, I hypothesize that it will be useful to have environments where you need to identify *what strategy your opponent is using*.\n", + "\n", + "In this environment, each strategy has the property that it beats *all* of the strategies that preceded it.\n", + "\n", + "As a result, it was fine for the agent to undergo catastrophic forgetting: even though it was trained against past agents, it only needed to learn the current strategy well; it didn’t need to remember previous strategies.\n", + "\n", + "As a result, it may have forgotten prior strategies and skills, which might have reduced its ability to learn new tasks quickly.\n", + "\n", + "**Read more:** Paper: Emergent Tool Use from Multi-Agent Autocurricula, Vox: Watch an AI learn to play hide-and-seek ### **Applications** Tackling Climate Change with Machine Learning *(David Rolnick et al)* (summarized by Rohin): See Import AI.\n", + " - Title: [AN #65]: Learning useful skills by watching humans \"play\", Author: Rohin Shah, Date published: 2019-09-23, URL: https://www.lesswrong.com/posts/GPADepj6yP8zqSbJh/an-65-learning-useful-skills-by-watching-humans-play\n", + "\n", + "It was a looping, meandering alley in the midst of Hogwarts, wandering like a stray lock of hair; sometimes crossing itself, it seemed, but you couldn’t ever get to the end if you gave into the temptation of apparent shortcuts. At the end of the tangle, six students leaned against rough stones, robes black against the grey walls and trimmed in green, eyes darting from one to each other. Torches burned in the windowless sconce, casting light to ward off the darkness and heat to ward off the chill of the Slytherin dungeons. \"I am *certain,*\" Reese Belka snapped, \"absolutely *certain,* that was no true ritual. Little firstie witches can’t do that kind of magic, and even if they could, who’s ever heard of a Dark ritual which *sacrifices* a sealed horror for—*that? *\" \"Were you—\" said Lucian Bole. \"I mean—after that girl snapped her fingers—\" Belka’s glare should have melted him. \"No,\" she spat, \"I was *not. *\" \"That is, she wasn’t naked,\" drawled Marcus Flint, his broad shoulders leaning back in apparent relaxation against the lumpy stone surface. \"Covered in chocolate frosting, yes, but not naked\". \"This day Potter has offered great insult to our Houses,\" said the grim voice of Jaime Astorga. \"Yes, well, I’m sorry to be blunt,\" Randolph Lee said evenly. The seventh-year duelist rubbed at his chin, where a faint fuzz of beard had been allowed to grow. \"But when someone sticks you to the ceiling, it’s a message, Astorga.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "It’s a message which says: I’m an incredibly powerful Dark Wizard who could’ve done anything to you I damn well pleased, and I don’t care if your House is offended, either\". Robert Jugson III gave a soft, low laugh at this, a chuckle that sent chills down several spines. \"It makes you wonder if you picked the wrong side, doesn’t it? I’ve heard tales about *messages* like that, sent at the old Dark Lord’s bidding...\" \"I’m not ready to kneel to Potter just yet,\" said Astorga, staring hard into Jugson’s eyes. \"Neither am I,\" said Belka. Jugson was holding his wand, and he turned it idly back and forth in his fingers, pointing it up and then downward. \"Are you a Gryffindor or a Slytherin\"? said Jugson. \"Everyone’s got a price. Everyone smart\". This statement produced a moment of silence. \"Shouldn’t Malfoy be here\"? Bole said tentatively. Flint gave a dismissive flick of his fingers. \"Whatever Malfoy’s plotting, he wants to put on an air of innocence. He can’t be seen missing at the same time as us\". \"But everyone *knows* that already,\" said Bole. \"Even in the other Houses\". \"Yes, very clumsy,\" said Belka. She snorted. \"Malfoy or no, he’s just a little firstie and we don’t need him here\". \"I will owl my father,\" Jugson said softly, \"and *he* will speak to Lord Malfoy himself—\" Abruptly, Jugson stopped speaking.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"I don’t know about *you*, dearies,\" Belka said with fake sweetness, \"but *I* don’t plan on running scared from a false ritual, and *I’m* not done with Potter and his pet mudblood\". Nobody answered. All their gazes were looking past her. Slowly, Belka turned around to see what the others were staring at. \"You will do *nothing,*\" hissed their Head of House. Severus Snape’s face was enraged, when he spoke small spots of spittle flew from his mouth, further dotting his already-dirtied robes. \"You fools have done *enough! * You have embarrassed my House—*lost* to first-years—now you speak of embroiling noble Lords of the Wizengamot in your *pathetic* childish squabbles? *I* shall deal with this matter. *You* will not embarrass this House again, you will not *risk* embarrassing this House again! You are *done* with fighting witches, and if I hear otherwise—\" If you thought they’d be sitting next to each other at dinnertime, after that, you’d be quite mistaken. \"What does she *want* from me\"? came the plaintive cry of a boy who, for all his extensive reading in the scientific literature, was still a bit naive about certain things. \"Did she *want* to get beaten up\"? The upper-year Ravenclaw boys who’d sat down next to him at the dinner-table exchanged swift glances with each other until, by some unspoken protocol, the most experienced of their number spoke.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"Look,\" said Arty Grey, the seventh-year who was leading in their competition by three witches and a Defense Professor, \"the thing you’ve got to understand is, just because she’s *angry* doesn’t mean you lost points. Miss Granger is angry because she got all frightened and you’re *there to be blamed,* you understand? But at the same time, even though she won’t admit it, she’ll be touched that her boyfriend went to such ridiculous and frankly insane lengths to protect her\". \"This is not about *points,*\" ground out Harry Potter, the words visibly escaping from between his clenched teeth. Dinner sat ignored on the table in front of him. \"This is about *justice. * And *I. Am. Not. Her. Boyfriend! *\" This was met by a certain amount of sniggering from all present. \"Yeah, well,\" said a sixth-year Ravenclaw boy, \"I think after she kisses you to bring you out of Dementation and you stick forty-four bullies to the ceiling for her, we’ve gone way past ‘she’s not my girlfriend, really’ and into the question of what your kids will be like. Wow, that’s a scary thought...\" The Ravenclaw trailed off and then said, in a smaller voice, \"Please don’t look at me like that\". \"Look,\" said Arty Grey, \"I’m sorry to be blunt about this, but you can have justice or you can have girls, you can’t have both at the same time\". He clapped a companionable hand on Harry Potter’s shoulder.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"You’ve got potential, kid, more potential than any wizard I’ve ever seen, but you’ve got to learn how to *use* it, you know? Be a bit sweeter to them, learn some spells to clean up that mess you call hair. Above all, you need to hide your evilness better—not *too* well, but better. Nice well-groomed boys get girls, and Dark Wizards also get girls, but nice well-groomed boys suspected of being *secretly* Dark get more girls than you can imagine—\" \"Not interested,\" Harry said flatly, as he picked up the boy’s hand from his shoulder and unceremoniously dropped it. \"But you will be,\" said Arty Grey, his voice low and foreboding. \"Ah, you will be\"! Elsewhere along the same table - \"*Romantic? *\" shrieked Hermione Granger, so loudly that some of the girls next to her winced. \"*What part of that was romantic? * He didn’t *ask! * He never *asks! * He just sends ghosts after people and glues them to ceilings and does whatever he wants with *my* life\"! \"But don’t you see\"? said a fourth-year witch. \"It means that even though he’s evil, he *loves* you\"! \"You’re not helping,\" said Penelope Clearwater a little further down the table, but she was ignored. Several older witches had started toward Hermione, after she’d sat down at the extreme opposite end of the table from Harry Potter, but then a swifter cloud of younger girls had surrounded Hermione in an impenetrable barrier. \"Boys,\" said Hermione Granger, \"should not be allowed to love girls without asking them first!\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "This is true in a number of ways and especially when it comes to gluing people to the ceiling\"! This was also ignored. \"It’s just like a play\"! sighed a third-year girl. \"A play\"? said Hermione. \"I’d like to see the play where anything like *this* happens\"! \"Oh,\" said the third-year girl, \"I was thinking of that really *romantic* one where there’s this very nice, sweet boy who makes a Floo call, only he mispronounces his destination and stumbles out into this room full of Dark Wizards who are performing a forbidden ritual that should’ve stayed forever lost to time, and they’re sacrificing seven victims in order to unseal this ancient horror which is supposed to grant someone a wish if it’s freed, so of course the boy’s presence interrupts the ritual, and as the horror is eating all the Dark Wizards and everyone is dying the boy’s last thought is that he wishes he could’ve had a girlfriend, and the next thing you know the boy is lying in the lap of this beautiful woman whose eyes are burning with a dreadful light, only she doesn’t understand anything about being human so the boy always has to stop her eating people. This is just like that play, only you’re the boy and Harry Potter is the girl\"! \"That...\" Hermione said, feeling quite surprised. \"That actually *does* sound something like—\" \"It *does? *\" blurted a second-year girl sitting across the table, who was now leaning forward, looking horrified and yet even more fascinated. \"No\"! said Hermione.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"I mean—*he’s not my boyfriend! *\" Two seconds later, Hermione’s ears caught up with what her lips had just said. The fourth-year witch put her hand on Hermione’s shoulder and gave her a comforting squeeze. \"Miss Granger,\" she said in a soothing voice, \"I think if you’re really honest with yourself, you’ll admit that the real reason you’re angry with your dark master is that he channeled his unspeakable powers through Tracey Davis instead of you\". Hermione’s mouth opened but her throat locked up before the words came out, which was probably a good thing, because if she’d actually yelled that loudly it would’ve broken something. \"How’s that possible, actually\"? said the third-year girl. \"I mean for Harry Potter to work through another girl even though he’s bound himself to you? Do the three of you have one of those, you know, arrangements\"? \"*Gaaaaack,*\" said Hermione Granger, her throat still locked, her brain halted, and her vocal cords spontaneously making a noise like she was coughing up a yak. *(Later. )* \"I don’t understand why you’re being so *unreasonable*,\" said another second-year witch, who’d replaced the third-year-girl after Hermione had threatened to ask Tracey to eat her soul. \"I mean, really, if someone like Harry Potter rescued *me*, I’d be—sending him thank-you cards, and hugging him, and,\" the girl’s face was a bit red, \"well, kissing him, I’d hope\". \"Yeah\"! said the other second-year witch.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"I’ve never understood why girls in plays get *angry* when the main character goes out of his way to be nice to them. *I* wouldn’t act like that if the hero liked *me. *\" Hermione Granger had dropped her head to the dinner table, her hands slowly pulling at her hair. \"You just don’t understand male psychology,\" the fourth-year witch said in an authoritative voice. \"Granger’s got to make it *look* like she can mysteriously resist his seductive charm\". *(Even later. )* And so before long Hermione Granger had turned to the only person left she could talk to, the only person guaranteed to understand her point of view - \"They’re all mad,\" said Hermione Granger as she strode vigorously toward Ravenclaw tower, having left dinner a bit early. \"Everyone except you and me, Harry, I mean *everyone* except us in this whole school of Hogwarts, they’re all entirely *mad. * And Ravenclaw girls are the *worst,* I don’t know *what* Ravenclaw girls go reading when they get older, but I’m certain they ought not to be reading it. One witch asked me if the two of us had soul-bonded, which I’m going to look up in the library tonight, but I’m pretty sure has never actually happened—\" \"I don’t even know a *name* for this kind of fallacious reasoning,\" said Harry Potter. The boy was walking normally, which meant he often had to skip forward a few steps to match her own indignation-fueled speed.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"I seriously think if it was up to *them*, they’d be dragging us off this minute to get our names changed to Potter-Evans-Verres-Granger… Ugh, saying that out loud makes me realize how awful it sounds\". \"You mean *your* name would be Potter-Evans-Verres-Granger and *mine* would be Granger-Potter-Evans-Verres,\" said Hermione. \"It’s too horrible to imagine\". \"No,\" said the boy, \"House Potter is a Noble House, so I think that name stays in front—\" \"*What? *\" she said indignantly. \"Who says *we* have to—\" There was a sudden awful silence, broken only by the thuds of their shoes. \"*Anyhow,*\" Hermione said hastily, \"some of the crazy things they said at dinner got me thinking, so I just want to say, Harry, that I really am grateful to you for saving me and everybody from getting beat up, and even though some parts of this afternoon upset me, I’m sure we can just talk about it calmly\". \"Ah...\" Harry said with a faint and tentative smile, his eyes showing a mixture of befuddlement and apprehension, \"that’s… good, I guess\"? To be specific, there’d been the fourth-year witch explaining that, since Harry was the evil wizard who’d fallen in love with Hermione, and Hermione was the pure and innocent girl who would either redeem him or get seduced by the Dark Arts herself, it followed that Hermione *had* to be perpetually indignant at anything Harry did, even if it was him heroically saving her from certain doom, just so that their romance wouldn’t resolve itself before the end of Act IV.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "And *then* Penelope Clearwater, who Hermione had really thought was smarter than that, had remarked in a loud voice that for identical reasons it was *impossible* for Hermione to just go over and talk sensibly with Harry about why she was feeling hurt, and anyway Dark Wizards were attracted to passionate defiance in a woman, not logic. This was the point at which Hermione had shoved herself up from the benches, stomped furiously over to where Harry was sitting, and asked him in a reasonable voice if the two of them could go for a walk and sort things out. \"So in other words,\" Hermione said in her calmest voice ever, \"you’re not really in trouble with me, I’m still talking to you, we’re still friends, and we’re still studying together. We’re *not* having a fight. Right\"? Somehow this only seemed to increase Harry Potter’s apprehension. \"Right,\" said the Boy-Who-Lived. \"Great\"! said Hermione. \"So, *have* you worked out why I was upset, Mr. Potter\"? There was a pause. \"You wanted me to keep out of your affairs\"? Harry said cautiously. \"I mean—I know you wanted to do things on your own. And I *was* staying out of your way, until I’d heard you’d gotten ambushed by three junior Death Eaters and, honestly, I wasn’t expecting that. *Professor Quirrell* wasn’t expecting that. I started to worry you’d gotten in over your head and then, no offense Hermione, forty-four bullies in a massed ambush is way beyond what *anyone* could handle without help.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "That’s why I thought you really needed help just that once—\" \"No, that part’s fine,\" said Hermione. \"We *were* in over our heads, honestly. Please guess again, Mr. Potter\". \"Um,\" said Harry. \"What Tracey did… startled you\"? \"Startled me, Mr. Potter\"? There might have been a touch of acidity in her voice. \"No, Mr. Potter, I was *scared. * I was *frightened. * I wouldn’t want to admit to being afraid of just *dragons* or something, people might think I was *cowardly*, but when you can hear distant voices crying ‘Tekeli-li! Tekeli-li! ’ and there’s pools of blood seeping out from under all the doors, then it’s okay to be scared\". \"I *am* sorry,\" Harry said with what sounded like genuine regret. \"I thought you’d realize it was me\". \"And the *reason* we all got scared like that, Mr. Potter, was that *you didn’t ask first! *\" Despite her intentions, Hermione found her voice was rising again. \"You should’ve *asked* me before you did something like that, Harry! You should’ve said very specifically, ‘Hermione, can I make blood come out from under the doors? ’ It’s important to be specific when you’re asking about that sort of thing\"! The boy rubbed the back of his neck as he walked. \"I… honestly, I just thought you’d *have* to say no\". \"Yes, Mr. Potter, *I could’ve said no*. That’s *the whole point of asking first*, Mr. Potter\"! \"No, I mean you’d have *had* to say no, whether or not it was what you *really* wanted.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "And then all of you would’ve gotten beaten up and it would’ve been *my* fault for asking first\". Hermione’s eyebrows went up in a bit of surprise, and she kept walking for a few steps while she tried to understand this. \"What\"? she said. \"Well...\" the boy said a bit slowly. \"I mean… you’re the Sunshine General, aren’t you? You *couldn’t* say yes to me scaring people, not even bullies, not even to save your friends from getting beaten up. You would’ve *had* to say no, and then you would’ve gotten hurt. This way, you can tell people honestly that you had no idea and that it wasn’t your fault. That’s why I didn’t warn you\". Hermione stopped walking, turned to face Harry full on instead of just turning her head. Her voice was carefully even as she said, \"Harry, you’ve *got* to stop coming up with clever reasons for doing stupid things\". Harry’s eyebrows flew up. After a moment he said, \"Look… I know what you mean, of course, but there’s still the question of whether it’s actually *is* a good idea, not just a clever one—\" \"I understand why you did what you did today,\" Hermione said. \"But I want you to promise that from now on, you’ll ask me first, always, even if you can come up with a reason why you shouldn’t\". There was a pause that stretched, and Hermione could feel her heart sinking. \"Hermione—\" Harry started to say. \"*Why? *\" The frustration burst out into her voice. \"*Why is it so awful? All you have to do is ask\"! * Harry’s eyes were very serious.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"Who in S.P.H.E.W. do you try hardest to defend, Hermione? Who are you most afraid for, when you fight\"? \"Hannah Abbott,\" Hermione said without having to think about it, and then felt a little bad, because Hannah *was* trying hard and she *had* improved a lot - \"Would you feel okay about trusting someone else, like Tracey, with *final* responsibility for protecting Hannah? If you knew Hannah was about to walk into an ambush, and you came up with a plan for protecting her, would you feel good about letting Tracey say whether or not you were allowed to do it\"? \"Well… no\"? said Hermione, puzzled. The green eyes of the Boy-Who-Lived were steady on hers. \"Would you trust *Hannah* to have the final say in whether she needed protecting\"? \"I—\" said Hermione, and then paused. It was strange, she knew the right answer and she also knew the right answer wasn’t actually true. Hannah was trying so hard to prove she wasn’t afraid, even though she *was,* and it was easy to see how the Hufflepuff girl might try *too* hard - Then Hermione realized the implication. \"You think I’m like *Hannah? *\" \"Not… exactly...\" Harry ran his hands through his mess of hair. \"Listen, Hermione, what would *you* have suggested doing, if I’d warned you about an ambush by forty-four bullies\"? \"I would’ve done the *responsible* thing and told *Professor McGonagall* and let *her* take care of it,\" Hermione said promptly.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"And *then* there wouldn’t have been darkness and people screaming and horrible blue light—\" But Harry just shook his head. \"That’s *not* the responsible thing to do, Hermione. It’s what someone playing the *role* of a responsible girl would do. *Yes*, I thought of going to Professor McGonagall. But she would’ve only stopped the disaster *once. * Probably before any disturbance happened in the first place, like by telling the bullies she knew. If the bullies got punished just for plotting, it would be by losing House points, or at worst a day’s detention, not anything that would really scare them. And then the bullies would have *tried again*. Fewer of them, with better operational security so I didn’t hear about it. They would probably ambush *one* of you, alone. Professor McGonagall doesn’t have the *authority* to do something scary enough to protect you—and *she* wouldn’t have overstepped her authority, because she’s not really responsible\". \"*Professor McGonagall* isn’t responsible\"? Hermione said incredulously. She jammed her hands on her hips, now openly glaring at him. \"Are you *nuts\"? * The boy didn’t blink. \"You could call it heroic responsibility, maybe,\" Harry Potter said. \"Not like the usual sort. It means that whatever happens, no matter what, it’s *always* your fault. Even if you tell Professor McGonagall, she’s not responsible for what happens, *you* are.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "Following the school rules isn’t an excuse, someone else being in charge isn’t an excuse, even trying your best isn’t an excuse. There just aren’t any excuses, you’ve got to *get the job done no matter what*\". Harry’s face tightened. \"That’s why I say you’re not thinking responsibly, Hermione. Thinking that your job is done when you tell Professor McGonagall—that isn’t heroine thinking. Like Hannah being beat up is *okay *then, because it isn’t *your fault *anymore. Being a heroine means your job isn’t finished until you’ve done* whatever it takes* to protect the other girls, *permanently. *\" In Harry’s voice was a touch of the steel he had acquired since the day Fawkes had been on his shoulder. \"You can’t think as if just following the rules means you’ve done your duty\". \"I think,\" Hermione said evenly, \"that you and I might disagree about some things, Mr. Potter. Like whether you or Professor McGonagall is more *responsible*, and whether being *responsible* usually involves people running around and screaming, and how much it’s a good idea to follow school rules. And just because we disagree, Mr. Potter, doesn’t mean that *you* get the final say\". \"Well,\" said Harry, \"you asked what was *so* awful about having to ask you first, and it was a surprisingly good question, so I examined my mind and that’s what I found.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "I think my real fear is that if Hannah is in trouble and I come up with a way to save her that seems weird or dark or something, you might not weigh the consequences to Hannah. You might not accept the heroine’s responsibility of coming up with *some* way to save her, somehow, no matter what. Instead you’d just carry out the *role* of Hermione Granger, the sensible Ravenclaw girl; and the *role* of Hermione Granger automatically says no, whether or not she has a better plan in mind. And then forty-four bullies will take turns beating up Hannah Abbott, and it’ll all be my fault because I *knew,* even if I didn’t want reality to be that way, I knew that was how it would go. I’m pretty sure that was my secret, wordless, unutterable fear\". The frustration was building up inside her again. \"It’s *my* life\"! Hermione burst out. She could imagine what it would be like with Harry messing with her all the time, constantly inventing justifications not to ask her first and not to listen to her objections. She shouldn’t have to *win an argument* just to—\"There’ll *always* be some reason, you can *always* say I’m not thinking right! I want my* own life! * Otherwise I’ll walk away, I really will, I mean it Harry\". Harry sighed. \"This is exactly where I didn’t want things to end up, and here we are. You’re afraid of just the same thing I am, aren’t you? Afraid that if *you* let go of the steering wheel, we’ll crash\". The corners of his lips twisted, but it didn’t look like a real smile.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"That’s something I can understand\". \"I don’t think you understand *at all! *\" Hermione said sharply. \"You said we’d be *partners,* Harry\"! That stopped him, she could see it stop him. \"How about this\"? Harry said at last. \"I’ll promise to ask you first before I do anything that could be interpreted as meddling in your affairs. Only *you’ve* got to promise *me* to be reasonable, Hermione. I mean *really*, genuinely, stop and think for twenty seconds first, treat it as a real choice. The sort of reasonableness where you realize I’m offering a way to protect the other girls, and that if you automatically say *no* without considering it properly, there’s this *actual consequence* where Hannah Abbott ends up in the hospital\". Hermione stared at Harry, as his recitation wound down. \"Well\"? said Harry. \"I shouldn’t have to make promises,\" she said, \"just to be *consulted* about *my own life. *\" She turned from Harry and began walking toward the Ravenclaw tower, not looking at him. \"But I’ll think about it, anyway\". She heard Harry sigh, and after that they walked in silence for a while, passing through an archway of some reddish metal like copper, into a corridor that was just like the one they’d left except that it was tiled in pentagons instead of squares. \"Hermione...\" said Harry. \"I’ve been watching you and thinking, since the day you said you were going to be a hero. You’ve *got* the courage.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "You’ll fight for what’s right, even in the face of enemies that would scare other people away. You’ve certainly got the raw intelligence for it, and you’re probably a better person inside than I am. But even so… well, to be honest, Hermione… I can’t quite see you filling Dumbledore’s shoes, leading magical Britain’s fight against You-Know-Who. Not yet, anyway\". Hermione had turned her head to stare at Harry, who just went on walking, as though lost in thought. Fill *those* shoes? She’d never tried to imagine herself that way. She’d never *imagined* imagining herself that way. \"And maybe I’m wrong,\" Harry said as they walked. \"Maybe I’ve just read too many stories where the heroes never do the sensible thing and follow the rules and tell their Professor McGonagalls, so my brain doesn’t think you’re a proper storybook hero. Maybe it’s you who’s the sane one, Hermione, and me who’s just being silly. But every time you talk about following rules or relying on teachers, I get that same feeling, like it’s bound up with this one last thing that’s stopping you, one last thing that puts your PC self to sleep and turns you into an NPC again...\" Harry let out a sigh. \"Maybe that’s why Dumbledore said I should have wicked stepparents\". \"He said *what? *\" Harry nodded. \"I still don’t know whether the Headmaster was joking or… the thing is, he was *right* in a way. I *had* loving parents, but I never felt like I could trust their decisions, they weren’t *sane* enough.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "I always knew that if I didn’t think things through myself, I might get hurt. Professor McGonagall will do whatever it takes to get the job done *if* I’m there to nag her about it, she doesn’t break rules on her own without heroic supervision. Professor Quirrell really *is* someone who gets things done no matter what, and he’s the only other person I know who notices stuff like the Snitch ruining Quidditch. But* him* I can’t trust to be *good*. Even if it’s sad, I think that’s part of the environment that creates what Dumbledore calls a hero—people who don’t have anyone else to shove final responsibility onto, and that’s why they form the mental habit of tracking everything themselves\". ** Hermione didn’t say anything to that, but she was thinking back to something Godric Gryffindor had written near the end of his very short autobiography. Briefly and without any explanation, because the scroll had been meant to be copied by hand, centuries before the Muggle printing press had inspired wizards to invent the Reading-Writing Quill. *No rescuer hath the rescuer,* Godric Gryffindor had written. *No Lord hath the champion, no mother and no father, only nothingness above. * If *that* was the price of being a hero, Hermione wasn’t sure she wanted to pay it. Or maybe—though it wasn’t the sort of thing she would have thought, before she started hanging around Harry—maybe Godric Gryffindor had gotten it *wrong*. \"Do you trust *Dumbledore\"? * Hermione said.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"I mean, he’s right here in our school and he’s the most legendary hero in the whole world—\" \"He *was* the most legendary hero,\" said Harry. \"Now he sets chickens on fire. Honestly, does Dumbledore seem reliable to *you? *\" Hermione didn’t answer. Side by side, the two of them began to climb huge wide spiral stairs, the steps alternating between bronze metal and blue stone; the final approach to where the Ravenclaw portrait waited to guard their dorm with silly riddles. \"Oh, and I just thought of something I should tell you,\" Harry said when they were about halfway up. \"Since it affects your life and all. Think of it as a sort of down payment—\" \"What is it\"? said Hermione. \"I predict S.P.H.E.W. is about to retire\". \"*Retire? *\" Hermione said, almost stumbling on one of the stairs. \"Yeah,\" Harry said. \"I mean, I could be wrong, but I suspect the teachers are about to clamp down hard on fighting in the corridors\". Harry was grinning as he spoke, a glint in his eyes behind the glasses hinting at secret knowledge. \"Cast new wards to detect offensive hexes, or start verifying reports of bullying using Veritaserum—I can think of several ways they might shut it down. But if I’m right, it’s something to celebrate, Hermione, you and all of you. You kicked up enough public ruckus that you got them to actually *do* something about the bullying. *All* the bullying\".\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "Slowly, then, a smile began to creep up her lips, and as she reached the top of the stairs and began walking toward the Ravenclaw portrait for her riddle, Hermione felt rather lighter on her feet, a wonderful lifting feeling spreading through her like she’d been pumped full of helium. Somehow, despite all the effort the eight of them had put in, she hadn’t expected *that* much, she hadn’t expected it to actually *work*. They’d made a *difference...* It was the end of breakfast-time on the next morning. The students from every year sat very still in their benches, all heads turned in the same direction, toward the Head Table, before which one lone first-year girl stood rigid and motionless, her head tilted back to stare up at the Head of House Slytherin. Professor Snape’s face was twisted with fury and triumph, vindictive as any painting of a Dark Wizard; and behind him the other Professors sat at the Head Table, watching with faces as though carved from stone. \"—permanently disbanded,\" spat the Potions Master. \"Your self-proclaimed Society is *outlawed* within Hogwarts, by my decision as a Professor! If your Society or any member of it is discovered fighting in the hallways again, Granger, you will be *personally* held responsible and expelled, by me, from the Hogwarts School of Witchcraft and Wizardry\"!\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "That first-year girl stood there, before the Head Table where she’d been called before only to receive commendations and smiles; stood there with her spine held tall and upright in its curve like a centaur’s bow, giving nothing to the enemy. That first-year witch stood there with all tears and anger bottled, her face still, nothing changing of her outward appearance, while something slowly broke inside her, she could feel it breaking. It broke further when Professor Snape gave her two weeks detention for the crime of violence in school, sneering with the contemptuous face he’d shown them all on the first day of Potions, and with a little twist in the corner of his smile that said the Potions Master knew exactly how unfair he was being. Whatever-it-was inside her cracked all the way through, from top to bottom, when Professor Snape took one hundred points from Ravenclaw. It ended, then, and Snape told her she was dismissed. She turned around and saw that at the Ravenclaw table, Harry Potter was sitting still in his place, she couldn’t see his expression from here, she saw his fists on the table but she couldn’t see if they were clenched white like her own. She had whispered to him, when Professor Snape had called her, that he wasn’t to do anything without asking first. Hermione wheeled back again to look at the Head Table, just as Snape was turning away from her to resume his place.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"I said you’re dismissed, girl,\" said the sneering voice, but there was a pleased smile on Snape’s face, like he was waiting for her to do something - Hermione strode forward another five steps toward the Head Table and said in a breaking voice, \"Headmaster\"? Utter silence filled the Great Hall. Headmaster Dumbledore said nothing, didn’t move. It was as though he, too, was just carved from stone. Hermione turned her gaze to look at Professor Flitwick, whose head, barely visible above the table, seemed to be staring down into his lap. Beside him, Professor Sprout’s face was very tight, she seemed to be forcing herself to watch, and her lips were trembling, but she said nothing. Professor McGonagall’s chair was empty, the Deputy Headmistress hadn’t shown up to breakfast that morning. \"Why aren’t any of you saying anything\"? said Hermione Granger. Her voice was trembling with the last of her hope, the last desperate reach for help from that place inside her. \"You *know* what he’s doing is wrong\"! \"Two more weeks’ detention, for insolence,\" Snape said silkily. It shattered. She looked at the Head Table for a few seconds longer, at Professor Flitwick and Professor Sprout and the empty place where Professor McGonagall should’ve been. Then Hermione Granger turned and began walking toward the Ravenclaw table. There was a babble of voices starting up, as the students came unfrozen from where they’d sat.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "And then, as she was almost to the Ravenclaw table - The dry voice of Professor Quirrell cut through everything, and that voice said, \"One hundred points to Miss Granger for doing what is right\". Hermione almost fell over her own feet; and then she continued forward, even as Snape shouted something furious, even as Professor Quirrell leaned back in his chair and began to laugh, even as Dumbledore’s voice was saying something she didn’t catch and then she was sitting down at the Ravenclaw table again next to Harry Potter. Harry Potter was frozen beside her, he looked like someone who didn’t dare move. \"It’s all right,\" her voice said to him, automatically without there being any choice or thought involved, although really it wasn’t right at all. \"But can you see if you can get me out of Snape’s detentions, like you did yourself that time\"? Harry Potter nodded, a single jerky motion of his head. \"I—\" said Harry. \"I—I’m sorry, this—this is all my fault—\" \"Don’t be *ridiculous*, Harry\". It was odd how her voice was coming out all normal, and without her thinking about what to say. Hermione looked down at her breakfast plate, but eating seemed to be clearly out of the question, there was a roiling and churning in her stomach which suggested that she was already on the verge of throwing up, which was odd because she could have sworn her whole body felt numb, like she wasn’t feeling anything, at the same time.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "\"And,\" her voice said, \"if you want to break school rules or something, you can ask me about it, I promise I won’t just say no\".\n", + "\n", + "Non est salvatori salvator,neque defensori dominus,nec pater nec mater,nihil supernum.\n", + " - Title: Chapter 75: Self Actualization Final, Responsibility, Author: Eliezer Yudkowsky, Date published: 2015-03-14, URL: https://www.lesswrong.com/posts/btfnD6vRQqcfMtuoX/chapter-75-self-actualization-final-responsibility\n", + "\n", + "Contents\n", + " - Attempted telekinesis\n", + " - The case of the munching noises\n", + " - The ad copy writer who doesn’t know if she’s \"good enough\"\n", + " - Useful \"telekinesis\": Separating babies from bathwater\n", + " - How to distinguish?\n", + " - Task type:\n", + " - Emotional tone:\n", + "Related to: Compartmentalization in epistemic and instrumental rationality; That other kind of status. ***Summary: ****I’d like to share some techniques that made a large difference for me, and for several other folks I shared them with. They are techniques for reducing stress, social shame, and certain other kinds of \"wasted effort\". These techniques are less developed and rigorous than the techniques that CFAR teaches in our workshops—for example, they currently only work for perhaps 1/​3rd of the dozen or so people I’ve shared them with—but they’ve made a large enough impact for that 1/​3rd that I wanted to share them with the larger group. I’ll share them through a sequence of stories and metaphors, because, for now, that is what I have. * For me, these techniques came out of a stressful time period. In October 2012, CFAR was very new, and I was very new to being its executive director. I was faced with a task that I basically didn’t know how to do—filling the first workshop for which we charged \"real\" money (the $3900/​person that actually let CFAR run), and helping our team create our first decently polished workshop at the same time (which needed curriculum, operations, etc. ).\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "But whenever I sat down to try to work, my head would fill up with all the other tasks I \"needed\" to get done, instead of the particular task I was trying to work on. Or my head would fill with stress and mental static. So, almost *because* of how badly I needed to work, I found myself unable to accomplish much of anything. The set of stories and metaphors below is somehow what eventually gave me the ability to work with full focus in those conditions (I found them partway through that October), and cured most of my decades-long social shame at the same time. [1] (Though, again, this stuff isn’t rigorous yet. It worked for a few folk, but failed a few others; your mileage may vary. Do share your thoughts. One morning, that month, I was lying in bed, half-asleep. And I wanted my laptop. But my laptop was a few feet away, so reaching it sounded hard (because I was half-asleep). \n", + "After lying there a while wishing, I finally noticed what my brain was up to. And I noticed that what my brain was doing was visualizing my laptop whooshing toward me. Again and again. (Fix attention on laptop… visualize the woosh. Nope, laptop isn’t here yet: repeat!)[2] I’m going to call this process \"Attempted telekinesis\". It seems to me that something like \"attempted telekinesis\" underlies a large set of stress /​ shame /​ worry /​ etc. , and that learning to vanish it has been super-useful for me and several others.\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "I’ll start with several examples of what I’ll be calling \"attempted telekinesis\", and then go into some techniques for vanishing it. Later that day, I was sitting at the office trying to work, and someone next to me was eating. Noisily. Now, I’m part of the sizable minority of the population that is driven absolutely bonkers by munching noises. Munching noises fill me with rage and make me want to punch someone. But, like, I get that that’s petty of me. So my internal thinking stream goes something like this: **Coworker: ** [Munch. Munch. ] **My system 1/​ intuitive brain*** (silently, in my head)***:** Argh! Stop it! **Me:** [Type, type. ] (While thinking: \"I don’t want to be petty; best not say anything, nor show annoyance on my face in any way\". ) *[1 minute later]* **Coworker:** [Munch. Munch. ] **My system 1/​ intuitive brain ***(silently, in my head)***:** Didn’t you hear me? ? Stop it! ! **Me:** [Type, type. ] (… I don’t want to be petty; best not say anything, or show it on my face in any way. ) *[and another minute later]* **Coworker:** [Munch. Munch. ] **My system 1/​ intuitive brain ***(silently, in my head)***:** Argh! ! Didn’t you hear me? ? Stop it! ! Why won’t it stop! ! Clearly I need to use even more emotional force to make it stop! ! **Me:** [no longer typing] (… Oh, huh, this is that \"attempted telekinesis\" thing again, isn’t it. I’m not doing anything with my face or voice that would cause the eating noises to cease.\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "I’m *intentionally* not doing anything with my face or voice, because I don’t want to be petty. So, later on that day, I sit down to write some ad copy—something I can email out to folks who might be interested in the workshop. And I notice that a bunch of my thoughts aren’t about the details of the ad wording at all—they’re about whether I’m good enough at writing to write ad copy, and also about whether the whole workshop is doomed and I’ll be cast desolate to the hyenas while my entire tribe mocks me for having ruined CFAR. So I stop and think through my fears for a moment. And I agree that, indeed, the workshop might not work—but since it also *might well* work, it’d be pretty damn stupid to stop preparing right then. And in fact, my useful \"next actions\" from this moment basically involve doing whatever’s most likely to make it work, and not wasting motions on the opposite prospect. Similarly, I might not be good enough at the writing—maybe I should be getting someone else to write it for me. But since I might well be able to, and since there’s no one good sitting right there to give the task to instead, it seems best to set a 1-hour timer, do the best writing I can for 1 hour without distracting myself trying to evaluate it—and then, when the timer rings, I can deliberately evaluate whether to write more myself or to look for someone else who can write it. But even after I think that through… my brain keeps on trying to waste these motions.\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "It’s like \"write… pause… `what if I’m not good enough? ’\". And I notice that it has the same feel as the laptop and the munching noises. As though something in me hopes that if I just feel upset about things, or if I just visualize that I need the world to be a certain way or worry about how it isn’t, this will somehow magic the world into a better state. A musical artist once said: \"You know, how good or bad you are [at making music] is really none of your goddamn business\". And I get what he meant, now. My business this hour is to write, not to worry about how I’m not good enough at writing. But how to do it? How to get my brain to focus on writing, and to drop the attempted telekinesis? # Useful \"telekinesis\": Separating babies from bathwater The \"attempted telekinesis\" examples above are all examples of pointless behaviors—the kinds of behaviors a person is better off removing. I’d like to take a moment, now, to distinguish pointless cases of \"attempted telekinesis\" (where a person tries to change the world just by repeatedly stressing out about it) from their useful cousins. Here’s a useful cousin: The other morning, I was lying in bed, again. Thinking that maybe I should get up. But feeling like bed was warm and getting up would be a little hard. And then I thought about breakfast. I pictured it: nice, fried eggs; a sliced fresh tomato; a steaming cup of tea. I pictured biting into the eggs, with the runny yolk on my tongue.\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "And suddenly, without any need of prompting from conscious-me, my body was in motion—up and heading toward the eggs. (Perhaps, from the perspective of the submodule of my brain that did the \"wishing for eggs\" manuever, wishing had in fact made it so! It wished, and my body responded: telekinetic success. ) As in the above \"pointless\" cases, my system 1 brain had a thing that it wanted, and visualized a picture of the desired end-state. But in the breakfast example, that visualization was useful. The imagined flying laptop had just filled my head with repeated wishing. The imagined vanishing munching noises had just filled my head with repeated aggravation. The imagined \"being a better writer\" state had only distracted me from writing. But the imagined experience of eating breakfast… pulled my system 1 into actually obtaining breakfast. Similarly, when I imagine Archimedes in the classic bathtub story, I imagine him obsessing a bit about how to measure the crown’s density. \"How can I measure it? How can I measure it\"? his brain might repeat… a little like repeating \"Get my laptop to whoosh toward me\"! . Obsessing on problems at CFAR certainly seems to help me notice potential solutions. So, what’s the take-away? When is it useful to try to wish the world into a different state? What distinguishes the kinds of \"attempted telekinesis\" that one might like to remove, from the kinds that fetch you breakfast or give you insights into the king’s crown?\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "This matter seems to me to be a bit complicated, but also seems quite important—if you get it wrong, you either stay unnecessarily distracted and ineffective (like me in the lead-up to CFAR’s first workshop), or you end up a sort of parody of pop Buddhism, sitting there being placid about your problems instead of harnessing your drives to solve them. ## How to distinguish? In practice, I tend to distinguish between useful and useless attempted telekinesis based on task type and emotional tone. (Improvements/​kibbitzes appreciated.) ## Task type: **Type 1: Problems that System 1 can solve by itself:** *Examples:* Making breakfast; causing someone to know you care about them. *Suggested response:* This sort of wishing is healthy, and may prompt actions that make a lot more sense than those system 2 would plan (e.g., your nonverbals as you apologize are likely to be far better if you viscerally care about your interlocutor). Leave system 1 be. **Type 2: Problems that are worth solving, but that need help from System 2:** *Examples:* \"There’s nothing good to eat\" (situation: you notice that several times, over the last hour, you’ve gone to the fridge, opened it, stared inside, closed it… and then opened it again a few minutes later—as though to see if something good has magically materialized into the closed fridge); Feeling ‘stuck’ at one’s job (or in a relationship); Not having enough money.\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "(The distinguishing feature here is that system 1 has been looping on the problem for a while to no effect, and that system 2 has not yet taken a good look at the problem.\n", + "\n", + "*Suggested response**:* Raise the problem to conscious attention; then, try to figure out what is bothering system 1; finally, decide what to do about it. As you do this, parts of the wishing will naturally shift from the general problem (\"Somehow make work less stuck-feeling\") to the specific strategy you’ve chosen (\"Figure out how to renegotiate with my manager\").[4]\n", + "\n", + "**Type 3: \"Problems\" that should be given up on:**\n", + "\n", + "*Examples:* \"Make the munching noises go away\" (in a case where you’ve decided not to); \"Make San Franciscans be better drivers\"; \"Let me vanish into the floor.\" (The distinguishing feature here is simply that these are \"problems\" that, on reflection, you do not wish to take action on.)\n", + "\n", + "*Suggested response**:* Find a way to let system 1 know that solving this problem isn’t worth the cost, or that keeping this problem on your internal \"worry/​fume about\" list is quite unlikely to have positive effects. For example, you might:\n", + "\n", + "\n", + " - Make a plan for what it would actually take to cause San Franciscans to be better drivers. Estimate the total amount of work involved. Ask your emotional brain if it would, in fact, like you to carry out this plan.\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + " - Visualize a stressed-out/​fuming/​worrying you getting cut off in traffic. Now visualize a calm you getting cut off in traffic. See if you expect to see anything good happen in the stressed-out case that doesn’t happen in the calm case. (Be open to the fact that the answer might be \"yes\".)[5]\n", + "\n", + " - Notice, in detail, what system 1 is upset about. Acknowledge that, yes, you may be late to your work meeting because of the traffic. And that, indeed, your personal driving habits are different from those of the driver who cut you off. And that someday a driver like that may in fact kill you via aggression or carelessness—it isn’t likely, but it’s possible, and the lifetime risk of death by traffic accident is distinctly nonzero. Once you’ve noticed all the painful things, check again to see whether it’s worth taking some sort of constructive action on some of them. System 1 may trust your policy decision more now that you’ve looked at all the downsides (and may be more willing, therefore, to stop trying to will the drivers into a different state).[6]\n", + "\n", + "**Type 4: Problems that should be delegated to a particular future-you:**\n", + "\n", + "*Examples:* The problem of locating a workshop venue (during the hour at which I was trying to write the workshops ad, that October); the situation with your roommates and the dishes (while you’re at work solving a coding problem).\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "\n", + "*Suggested response**:* Designate a particular future-you to do the task. Dialog with your \"inner simulator\" (your system 1 anticipations) until both system 1 and system 2 are convinced that that specific you will actually do the task, and that there is no additional positive effect to be gained via staying preoccupied now. **Type 5: Problems that System 2 needs \"shower-thoughts\" help with:** *Examples:* Archimedes’ problem measuring the king’s crown; \"My relationship with Fred is broken, and I can’t figure out what to do about it\"; \"How the heck can I solve that math riddle?\" (The distinguishing feature here is that both: (1) the problem has already been raised to conscious attention at some point (and system 2 failed to instantly solve it); and (2) the problem is a worthy use of your shower-thoughts—either for what it’ll accomplish directly, or for the improvement it may give to your pattern of thought.)) *Suggested response**:* This sort of wishing is healthy. Leave system 1 be. ## Emotional tone: Wishes often seem to me to have emotional tones. Some tones are simple desire (\"Breakfast… mmm...\". ). \n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "It seems to me that it’s worth installing an \"alert\" that sounds, in your head, whenever it hears either the hopeless/​bitter/​resigned tone, or the flailing/​save-me tone. Both are often signs of buggy \"attempted telekinesis\" situations that are worth conscious debugging (a la the schema above). And the emotional tones can be easier to automatically flag. [1] A book called \"Bonds that make us free\" played a substantial role in prompting these thoughts and was extremely helpful to me. It’s written from a Christian worldview, but if you’re up for navigating a foreign expository style and sorting out for yourself which parts to keep, and if in addition you are interested in vanishing social shame or other forms of loopy thoughts, I’d recommend it. [2] Thanks to Alicorn for making the cartoon. [3] Other than, you know, to repeatedly visualize my thought patterns whooshing into the new state that I now wished them to be in? ;) [4] The book \"Focusing\" by Eugene Gendlin teaches one useful way to do this. If you decide to check it out, I’d strongly recommend the audiobook over the paper book, as it is abridged and far clearer. [5] For example, perhaps, if you remain stressed out, perhaps your boss will see how much you suffered in your attempt to be on time to work and will deduce that you care about timeliness. (If you notice some good effect coming from the stress that doesn’t come from the calm, you might want to look for an alternative way to cause the effect.\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "For example, you might update your heuristics to decrease the chances of future lateness; plan to explain this to your boss and to offer a 1-sided $100 bet against this ever happening again; and then drive with your mind free to focus fully on interesting problems.\n", + "\n", + ") [6] More generally, when setting out to convince system 1 that X is true, it is best to be honestly curious as to whether X might in fact be false, and whether system 1 may have some good reason for suspecting this.\n", + "\n", + "It is much the same as when attempting to convince another human.\n", + "\n", + "Saying \"Hey, look, you’re wrong and stupid and so your proposed office policy is really bad\" is usually not very persuasive; saying \"huh; I’m confused; the office policy looks to me as though it’ll cost a lot of hours to little effect, but you usually have good reasons for things; maybe you could tell me why you think it’s plausible\"?\n", + "\n", + "is often a better way to persuade; one wishes to do the same thing for system 1.\n", + " - Title: Attempted Telekinesis, Author: AnnaSalamon, Date published: 2015-02-07, URL: https://www.lesswrong.com/posts/eRohP4gbxuBuhqTbe/attempted-telekinesis\n", + "\n", + "## Discussion article for the meetup : Tel Aviv: Hardware Verification and FAI\n", + "\n", + "\n", + "\n", + "**WHEN:**\n", + "28 October 2015 12:59:48AM (+0300)\n", + "\n", + "\n", + "\n", + "**WHERE:**\n", + "Electra Tower\n", + "\n", + "\n", + "\n", + "\n", + "We will meet at Google Israel on the 29th floor, as always.\n", + "\n", + "The speaker this time is Yoav Hollander, inventor of the \"e\" hardware verification language and founder of Verisity. His description of the talk:\n", + "\n", + "\"I’ll (briefly) describe the FAI verification problem, and admit that I don’t really know how to solve it. I’ll also warn against ‘magical thinking’, i.e. assuming that because a fool-proof solution is needed, it will somehow appear before the window of opportunity slams on our finger tips.\n", + "\n", + "I’ll review what works (and what does not) in HW verification and in autonomous systems verification, and discuss why some of that may be relevant for FAI verification.\n", + "\n", + "I’ll then open the room for discussion.\"\n", + "\n", + "Facebook event: https://​​www.facebook.com/​​events/​​907241922691991/​​\n", + "My phone: 0542600919 (Vadim)\n", + "\n", + "\n", + "## Discussion article for the meetup : Tel Aviv: Hardware Verification and FAI\n", + "\n", + " - Title: Meetup : Tel Aviv: Hardware Verification and FAI, Author: Squark, Date published: 2015-10-01, URL: https://www.lesswrong.com/posts/HbXYSWHnaXWhnJm7B/meetup-tel-aviv-hardware-verification-and-fai\n", + "\n", + "(I saw October didn’t have one. First post—please let me know if I do something wrong.To whoever comes after me: Yoav Ravid comments that the wording could use an update.)If it’s worth saying, but not worth its own post, here’s a place to put it.\n", + "If you are new to LessWrong, here’s the place to introduce yourself. Personal stories, anecdotes, or just general comments on how you found us and what you hope to get from the site and community are invited. This is also the place to discuss feature requests and other ideas you have for the site, if you don’t want to write a full top-level post.\n", + "If you want to explore the community more, I recommend reading the Library, checking recent Curated posts, seeing if there are any meetups in your area, and checking out the Getting Started section of the LessWrong FAQ. If you want to orient to the content on the site, you can also check out the new Concepts section.\n", + "The Open Thread tag is here. The Open Thread sequence is here.\n", + "\n", + " - Title: Open & Welcome Thread October 2021, Author: qazzquimby, Date published: 2021-10-04, URL: https://www.lesswrong.com/posts/HjFkEcw26GGHrjMXu/open-and-welcome-thread-october-2021\n", "\n" ] } @@ -974,7 +2674,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -984,20 +2684,31 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ + "\"\"\"\n", + "TODO:\n", + "Add a moderation call to not be prompt-hacked: https://platform.openai.com/docs/guides/moderation/quickstart\n", + "\n", + "\"\"\"\n", + "\n", "class AlignmentSearch:\n", " def __init__(self,\n", " dataset: Dataset, # Dataset object containing the data.\n", " ):\n", " self.dataset = dataset\n", " \n", - " @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(10))\n", + " # @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(10))\n", " def get_embedding(self, text: str) -> np.ndarray:\n", - " result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n", - " return result[\"data\"][0][\"embedding\"]\n", + " try:\n", + " result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n", + " return result[\"data\"][0][\"embedding\"]\n", + " except openai.RateLimitError as e:\n", + " print(\"Rate limit exceeded. Retrying in 30 seconds.\")\n", + " time.sleep(30)\n", + " return self.get_embedding(text)\n", " \n", " def get_top_k(self, query: str, k: int=10) -> List[str]:\n", " # Receives a query (str) and returns the top k blocks that are most semantically similar to the query.\n", @@ -1008,12 +2719,13 @@ " top_k = [self.dataset.embed_split[i] for i in top_k_indices]\n", " return top_k\n", " \n", - " def construct_messages(self, question: str, blocks: List[str] = [\"None\"], mode: str = \"balanced\") -> str:\n", + " def construct_messages(self, question: str, blocks: List[str] = None, mode: str = \"balanced\") -> str:\n", " # Receives a question (str) and a list of blocks and returns a prompt (str) to be used for text generation.\n", - " context = \"\"\n", - " for i, block in enumerate(blocks):\n", - " context += f\"Context #{i+1}: {block}\\n\\n\"\n", - " context = context[:MAX_LEN_PROMPT * 3] + \"...\" if len(context) > MAX_LEN_PROMPT * 3 else context[:-2]\n", + " if blocks:\n", + " context = \"\"\n", + " for i, block in enumerate(blocks):\n", + " context += f\"Context #{i+1}: {block}\\n\\n\"\n", + " context = context[:MAX_LEN_PROMPT * 3] + \"...\" if len(context) > MAX_LEN_PROMPT * 3 else context[:-2]\n", " \n", " if mode == \"balanced\":\n", " assistant_prompt = \"You are a helpful assistant, and you help users by answering questions and providing information about AI Alignment and AI Safety. You are extremely knowledgeable, yet you know the limits of your own knowledge. Answer the user's questions as truthfully as possible using the provided context, and if the answer is not contained within it, say \\\"I don't know.\\\", or \\\"I'm not sure I know the answer to your question. However, I can try.\\\" followed by an attempt to answer as best you can. You can also ask the user questions to clarify their question.\"\n", @@ -1053,141 +2765,38 @@ " model=COMPLETIONS_MODEL, \n", " messages=messages\n", " )\n", - " top_k = self.get_top_k(hyde_completion, k)\n", + " top_k = self.get_top_k(f\"{question}\\n{hyde_completion}\", k)\n", + " # print(top_k)\n", + " # raise NotImplementedError\n", " else:\n", " top_k = self.get_top_k(question, k)\n", " answer = self.answer_question(question, top_k)\n", - " return answer" + " return answer, top_k, sources" ] }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"Not necessarily. While it's possible that smarter AI could have a better understanding of ethics and morality, the relationship between intelligence and morality is not straightforward. Ethical behavior requires more than just intelligence; it requires values, empathy, and an understanding of complex human situations. Moreover, a smarter AI could use its intelligence to achieve goals that conflict with human values or lead to unintended consequences. Ensuring that AI is aligned with human values and ethical principles is a major focus of AI safety research.\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1678851324,\n", - " \"id\": \"chatcmpl-6uCD6zNQj230yYrUdYCHoUJ2Vabv1\",\n", - " \"model\": \"gpt-3.5-turbo-0301\",\n", - " \"object\": \"chat.completion\",\n", - " \"usage\": {\n", - " \"completion_tokens\": 98,\n", - " \"prompt_tokens\": 3256,\n", - " \"total_tokens\": 3354\n", - " }\n", - "}\n", - "{\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"Not necessarily. While it's possible that smarter AI could have a better understanding of ethics and morality, the relationship between intelligence and morality is not straightforward. Ethical behavior requires more than just intelligence; it requires values, empathy, and an understanding of complex human situations. Moreover, a smarter AI could use its intelligence to achieve goals that conflict with human values or lead to unintended consequences. Ensuring that AI is aligned with human values and ethical principles is a major focus of AI safety research.\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1678851324,\n", - " \"id\": \"chatcmpl-6uCD6zNQj230yYrUdYCHoUJ2Vabv1\",\n", - " \"model\": \"gpt-3.5-turbo-0301\",\n", - " \"object\": \"chat.completion\",\n", - " \"usage\": {\n", - " \"completion_tokens\": 98,\n", - " \"prompt_tokens\": 3256,\n", - " \"total_tokens\": 3354\n", - " }\n", - "}\n" + "This claim is controversial and not widely accepted in the field of AI Alignment. While it is possible that smarter AI will have a better understanding of morality, it is not guaranteed, and there are reasons to believe that AI may not have human-like values or morals. One of the major challenges in AI Alignment is ensuring that AI systems pursue objectives that align with human values and goals, which may not be straightforward for AI systems to understand or infer. Therefore, research in AI Alignment is important for ensuring that AI is developed and used in ways that are safe, ethical, and beneficial for humanity.\n" ] } ], "source": [ "SA = AlignmentSearch(dataset=dataset)\n", - "query = \"Is it true that smarter AI will also be more moral?\"\n", - "# top_k = SA.get_top_k(query, 10)\n", - "# print(top_k)\n", + "query = \"Claim: AI Alignment is unnecessary because smarter AI will also be more moral.\"\n", "answer = SA.search_and_answer(query, 10)#, HyDE=True)\n", "print(answer)" ] }, { "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "It’s often not stated as an assumption; rather, inferences are made assuming that you have the background model that the AI is goal-directed. This makes it particularly hard to question the assumption, since you don’t realize that the assumption is even there. Another reason that this assumption is so easily accepted is that we have a long history of modeling rational agents as expected utility maximizers, and for good reason: there are many coherence arguments that say that, *given that you have preferences/​goals*, if you aren’t using probability theory and expected utility theory, then you can be taken advantage of. It’s easy to make the inference that a superintelligent agent must be rational, and therefore it must be an expected utility maximizer.\n", - " - Title: More marbles and Sleeping Beauty, Author: Manfred, Date published: 2014-11-23, URL: https://www.lesswrong.com/posts/c7fAEStCafGh8TFdf/more-marbles-and-sleeping-beauty\n", - "\n", - " - A goal-conditioned policy with common sense could be operationalized as optimizing for the goal of \"following a human’s orders without doing anything that humans would reliably judge as crazy\". - MIRI’s version of corrigibility seems like it stays within this framework. - You could think of the services in CAIS as optimizing for the *aggregate* reward they get over all time, rather than just the reward they get during the current episode. I do *not* want these versions of the scenarios, since they then make it tempting to once again say \"but if you get the goal even slightly wrong, then you’re in big trouble\".\n", - " - Title: Sydney Rationality Dojo—November, Author: Cpt. Bl, Date published: 2019-01-02, URL: https://www.lesswrong.com/events/tMzE2FPbHroAJ54L4/sydney-rationality-dojo-november\n", - "\n", - "Because this assumption was so embedded in how I thought about the problem, I had trouble imagining how else to even consider the problem. I would guess this is true for at least some other people, so I want to summarize the counterargument, and list a few implications, in the hope that this makes the issue clearer. ### Why goal-directed behavior may not be required The main argument of this chapter is that it is not *required* that a superintelligent agent takes actions in pursuit of some goal. It is possible to write algorithms that select actions without doing a search over the actions and rating their consequences according to an explicitly specified simple function.\n", - " - Title: LessWrong Study Hall will be password-protected, Author: tkadlubo, Date published: 2013-11-13, URL: https://www.lesswrong.com/posts/BCvED5FkJ9GBAwQ9M/lesswrong-study-hall-will-be-password-protected\n", - "\n", - "They might all read a few books worth of stuff, but they won’t all have read the same books. The information that they can be coordinated around is more like \"several blogposts\". If you’re trying to coordinate *nerds*, maybe those blogposts add up to one book because nerds like to read. If you want to coordinate 1,000 people… you realistically get one blogpost, or maybe one blogpost worth of jargon that’s hopefully self-explanatory enough to be useful. If you want to coordinate thousands of people... You have about five words. This has ramifications on how complicated a coordinated effort you can attempt. What if you *need* all that nuance *and* to coordinate thousands of people?\n", - " - Title: Sydney Rationality Dojo—November, Author: Cpt. Bl, Date published: 2019-01-02, URL: https://www.lesswrong.com/events/tMzE2FPbHroAJ54L4/sydney-rationality-dojo-november\n", - "\n", - "As it turns out, nope, it’s the same exact game, just re-labeled.\n", - "\n", - "In the re-labeled marble game you still have two unknown variables (represented by flipping coins), and you still have a 1⁄2 chance of black and Tails, a 1⁄4 chance of black and Heads, and a 1⁄4 chance of white and Heads.\n", - "\n", - "And then to get the thirds, you ask the question \"If I get a black marble, what is the probability of the faces of the first coin?\" Now you update to P(Heads|black)=1/​3 and P(Tails|black)=2/​3.\n", - "\n", - "**II**\n", - "\n", - "Okay, enough analogies. What’s going on with these two positions in the Sleeping Beauty problem?\n", - "\n", - "**1:** ** 2:**\n", - " - Title: More marbles and Sleeping Beauty, Author: Manfred, Date published: 2014-11-23, URL: https://www.lesswrong.com/posts/c7fAEStCafGh8TFdf/more-marbles-and-sleeping-beauty\n", - "\n", - "Because this assumption was so embedded in how I thought about the problem, I had trouble imagining how else to even consider the problem. I would guess this is true for at least some other people, so I want to summarize the counterargument, and list a few implications, in the hope that this makes the issue clearer. ### Why goal-directed behavior may not be required The main argument of this chapter is that it is not *required* that a superintelligent agent takes actions in pursuit of some goal. It is possible to write algorithms that select actions without doing a search over the actions and rating their consequences according to an explicitly specified simple function.\n", - " - Title: Southern California FAI Workshop, Author: Scott Garrabrant, Date published: 2014-04-20, URL: https://www.lesswrong.com/posts/nqgY7mNy8JMN99SHA/southern-california-fai-workshop\n", - "\n", - "Once we let go of the idea of optimizing for a single goal and it becomes possible to think about other ways in which we could build AI systems, there are more insights about how we could build an AI system that does what we intend instead of what we say.\n", - "\n", - "(In my case it was reversed—I heard a lot of good insights that don’t fit in the framework of goal-directed optimization, and this eventually led me to let go of the assumption of goal-directed optimization.\n", - "\n", - ") We’ll explore some of these in the next chapter.\n", - " - Title: [Link] Truth-telling is aggression in zero-sum frames (Jessica Taylor), Author: ioannes, Date published: 2019-09-11, URL: https://www.lesswrong.com/posts/T39zJNzXEeh9MR47h/link-truth-telling-is-aggression-in-zero-sum-frames-jessica\n", - "\n", - "### Implications At a high level, I think that the main implication of this view is that we should be considering other models for future AI systems besides optimizing over the long term for a single goal or for a particular utility or reward function. - G*oal-conditioned policy with common sense:* In this setting, humans can set goals for the AI system simply by asking it in natural language to do something, and the AI system sets out to do it. However, the AI also has \"common sense\", where it interprets our commands pragmatically and not literally: it’s not going to prevent us from setting a new goal (which would stop it from achieving its current goal), because common sense tells it that we don’t want it to do that.\n", - " - Title: [Link] Truth-telling is aggression in zero-sum frames (Jessica Taylor), Author: ioannes, Date published: 2019-09-11, URL: https://www.lesswrong.com/posts/T39zJNzXEeh9MR47h/link-truth-telling-is-aggression-in-zero-sum-frames-jessica\n", - "\n", - " - A goal-conditioned policy with common sense could be operationalized as optimizing for the goal of \"following a human’s orders without doing anything that humans would reliably judge as crazy\". - MIRI’s version of corrigibility seems like it stays within this framework. - You could think of the services in CAIS as optimizing for the *aggregate* reward they get over all time, rather than just the reward they get during the current episode. I do *not* want these versions of the scenarios, since they then make it tempting to once again say \"but if you get the goal even slightly wrong, then you’re in big trouble\".\n", - " - Title: Southern California FAI Workshop, Author: Scott Garrabrant, Date published: 2014-04-20, URL: https://www.lesswrong.com/posts/nqgY7mNy8JMN99SHA/southern-california-fai-workshop\n", - "\n", - "One way to think about this is to consider an AI system that infers and follows human *norms*, which are probably much easier to infer than human values (most humans seem to infer norms very accurately). - *Corrigible AI:* I’ll defer to Paul Christiano’s explanation of corrigibility. - *Comprehensive AI Services (CAIS):* Maybe we could create lots of AI services that interact with each other to solve hard problems. Each individual service could be bounded and episodic, which immediately means that it is no longer optimizing over the long term (though it could still be goal-directed). \n", - " - Title: Decentralization How—Los Angeles LW/​SSC Meetup #74 (Wednesday, September 12th), Author: T3t, Date published: 2018-09-12, URL: https://www.lesswrong.com/events/fbhEipkzZPwfFYPD3/decentralization-how-los-angeles-lw-ssc-meetup-74-wednesday\n", - "\n" - ] - } - ], - "source": [ - "for k in top_k:\n", - " print(k)\n", - " print()" - ] - }, - { - "cell_type": "code", - "execution_count": 147, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -1195,14 +2804,22 @@ "output_type": "stream", "text": [ "Source Truth Empirical Difference\n", - "total 41614 14 41600 \n", - "lesswrong 28479 11 28468 \n", - "alignment forum 2138 1 2137 \n", - "arxiv 8007 2 8005 \n", + "alignment forum 2138 105 2033 \n", + "manual ? 0 UNKNOWN \n", + "waitbutwhy.com 2 0 2 \n", + "https://aiimpacts.or 227 11 216 \n", + "arbital.com 223 10 213 \n", + "https://intelligence 479 22 457 \n", + "reports ? 2 UNKNOWN \n", + "https://aisafety.cam 8 0 8 \n", + "curriculum ? 0 UNKNOWN \n", + "https://www.yudkowsk 23 2 21 \n", + "distill 49 0 49 \n", + "total 41614 152 41462 \n", "\n", " Truth Empirical Difference\n", - "Word Count 53550146 23344 53526802 \n", - "Character Count 351767163 143048 351624115 \n" + "Word Count 53550146 306879 53243267 \n", + "Character Count 351767163 1893590 349873573 \n" ] } ], @@ -1251,6 +2868,14 @@ "print(f\"{'Character Count':<20} {char_count_truth:<10} {dataset.total_char_count:<10} {char_count_truth - dataset.total_char_count:<10}\")" ] }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tests" + ] + }, { "cell_type": "code", "execution_count": 5, @@ -1328,7 +2953,24 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "try:\n", + " response = openai.ChatCompletion.create(\n", + " model=kConfig.model,\n", + " messages=messages,\n", + " stream=True)\n", + " with Live(md, auto_refresh=False) as lv:\n", + " for part in response:\n", + " finish_reason = part[\"choices\"][0][\"finish_reason\"]\n", + " if \"content\" in part[\"choices\"][0][\"delta\"]:\n", + " content = part[\"choices\"][0][\"delta\"][\"content\"]\n", + " answer += content\n", + " md.markup = answer\n", + " md.parsed = parser.parse(md.markup)\n", + " lv.refresh()\n", + " elif finish_reason:\n", + " pass\n" + ] } ], "metadata": {