diff --git a/src/Embeddings Search/testing.ipynb b/src/Embeddings Search/testing.ipynb index f3f0a46..ed5486f 100644 --- a/src/Embeddings Search/testing.ipynb +++ b/src/Embeddings Search/testing.ipynb @@ -103,13 +103,13 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "import jsonlines\n", "import numpy as np\n", - "from typing import List, Dict, Tuple\n", + "from typing import List, Dict, Tuple, DefaultDict\n", "import re\n", "import time\n", "import random\n", @@ -124,12 +124,38 @@ "import tiktoken\n", "\n", "import config\n", - "from pathlib import Path" + "from pathlib import Path\n", + "from collections import defaultdict" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "defaultdict(int, {'a': 1})" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test = defaultdict(int)\n", + "test[\"a\"] += 1\n", + "type(test)\n", + "test: DefaultDict[str, int] = defaultdict(int)\n", + "test[\"a\"] += 1\n", + "test" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -272,7 +298,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -285,7 +311,8 @@ "\n", "MAX_LEN_PROMPT = 5000\n", "\n", - "project_path = Path.cwd().parent.parent\n", + "project_path = Path.cwd().parent.parent # .cwd() returns the current working directory, \n", + "# but in a notebook it is the directory of the notebook. So we need to go up two levels.\n", "PATH_TO_DATA = project_path / \"src\" / \"Embeddings Search\" / \"data\" / \"alignment_texts.jsonl\" # Path to the dataset .jsonl file.\n", "PATH_TO_EMBEDDINGS = project_path / \"src\" / \"Embeddings Search\" / \"data\" / \"embeddings.npy\" # Path to the saved embeddings (.npy) file.\n", "PATH_TO_DATASET = project_path / \"src\" / \"Embeddings Search\" / \"data\" / \"dataset.pkl\" # Path to the saved dataset (.pkl) file.\n" @@ -301,7 +328,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -319,7 +346,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -334,49 +361,55 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ + "\n", + "\n", + "\n", "class Dataset:\n", + " \n", " def __init__(self,\n", - " path: str, # Path to the dataset .jsonl file.\n", - " sources: List[str] = None, # List of sources to include. If None, include all sources.\n", + " jsonl_data_path: str, # Path to the dataset .jsonl file.\n", + " custom_sources: List[str] = None, # List of sources to include, like \"alignment forum\", \"lesswrong\", \"arxiv\",etc.\n", " rate_limit_per_minute: int = 3_500, # Rate limit for the OpenAI API.\n", - " block_min_max_size: Tuple[int, int] = None, # Tuple of (min_block_size, max_block_size), used for the text splitter. If None, use default values.\n", + " min_tokens_per_block: int = 400, # Minimum number of tokens per block.\n", + " max_tokens_per_block: int = 600, # Maximum number of tokens per block.\n", " fraction_of_articles_to_use: float = 1.0, # Fraction of articles to use. If 1.0, use all articles.\n", " ):\n", - " self.path = path\n", - " self.sources = sources\n", + " self.jsonl_data_path = jsonl_data_path\n", + " self.custom_sources = custom_sources\n", " self.rate_limit_per_minute = rate_limit_per_minute\n", " self.delay_in_seconds = 60.0 / self.rate_limit_per_minute\n", " self.fraction_of_articles_to_use = fraction_of_articles_to_use\n", " \n", " # Set up text splitter\n", - " if block_min_max_size is None: self.block_min_max_size = (400, 600)\n", - " else: self.block_min_max_size = block_min_max_size\n", - " self.text_splitter = TokenSplitter(min_tokens=self.block_min_max_size[0], max_tokens=self.block_min_max_size[1])\n", + " self.min_tokens_per_block = min_tokens_per_block\n", + " self.max_tokens_per_block = max_tokens_per_block\n", " \n", - " self.data: List[Tuple[str]] = [] # List of tuples, each containing the title of an article, its URL, and text. E.g.: [('title', 'url', 'text'), ...]\n", - " self.embed_split: List[str] = [] # List of strings, each being a few paragraphs from a single article (not exceeding 1000 words).\n", + " self.metadata: List[Tuple[str]] = [] # List of tuples, each containing the title of an article, its URL, and text. E.g.: [('title', 'url', 'text'), ...]\n", + " self.embedding_strings: List[str] = [] # List of strings, each being a few paragraphs from a single article (not exceeding 1000 words).\n", " \n", - " self.num_articles: Dict[str, int] = {} # Number of articles per source. E.g.: {'source1': 10, 'source2': 20, 'total': 30}\n", - " if sources is None:\n", - " self.num_articles['total'] = 0\n", - " else:\n", - " for source in sources: \n", - " self.num_articles[source] = 0\n", - " self.num_articles['total'] = 0\n", + " self.articles_count: DefaultDict[str, int] = defaultdict(int) # Number of articles per source. E.g.: {'source1': 10, 'source2': 20, 'total': 30}\n", + "\n", + " if self.custom_sources is not None:\n", + " for source in self.custom_sources:\n", + " self.articles_count[source] = 0\n", + " self.total_articles_count = 0\n", " \n", " self.total_char_count = 0\n", " self.total_word_count = 0\n", " self.total_sentence_count = 0\n", " self.total_block_count = 0\n", " \n", + " self.sources_so_far: List[str] = []\n", + " self.info_types: Dict[str, List[str]] = {}\n", + "\n", " def get_info_tmp(self):\n", " self.sources_so_far = []\n", " self.info_types: Dict[str, List[str]] = {}\n", - " with jsonlines.open(self.path, \"r\") as reader:\n", + " with jsonlines.open(self.jsonl_data_path, \"r\") as reader:\n", " for entry in reader:\n", " if 'source' not in entry: entry['source'] = 'None'\n", " \n", @@ -400,23 +433,32 @@ " \"\"\"\n", " \n", " def get_alignment_texts(self):\n", - " with jsonlines.open(self.path, \"r\") as reader:\n", + " text_splitter = TokenSplitter(self.min_tokens_per_block, self.max_tokens_per_block)\n", + " with jsonlines.open(self.jsonl_data_path, \"r\") as reader:\n", " for entry in reader:\n", " try:\n", " if 'source' not in entry: raise MissingDataException(\"Entry has no source.\")\n", + " \n", + " \n", " random_number = random.random()\n", " if random_number > self.fraction_of_articles_to_use:\n", " continue\n", " \n", - " if self.sources is None:\n", - " if entry['source'] not in self.num_articles: self.num_articles[entry['source']] = 1\n", - " else: self.num_articles[entry['source']] += 1\n", - " self.num_articles['total'] += 1\n", + " # if we specified custom sources, only include articles from those sources\n", + " if (self.custom_sources is not None) and (entry['source'] not in self.custom_sources):\n", + " continue\n", + " self.articles_count[entry['source']] += 1\n", + " self.total_articles_count += 1\n", + " \n", + " \"\"\"if self.sources is None:\n", + " if entry['source'] not in self.articles_count: self.articles_count[entry['source']] = 1\n", + " else: self.articles_count[entry['source']] += 1\n", + " self.total_articles_count += 1\n", " else:\n", " if entry['source'] in self.sources:\n", - " self.num_articles[entry['source']] += 1\n", - " self.num_articles['total'] += 1\n", - " else: continue\n", + " self.articles_count[entry['source']] += 1\n", + " self.total_articles_count += 1\n", + " else: continue\"\"\"\n", " \n", " text=title=author=citation_level=date_published=url=tags=None\n", " \n", @@ -462,6 +504,7 @@ " elif type(entry['tags']) == str: tags = entry['tags']\n", " else: tags = None\n", " \n", + " # Get signature\n", " signature = \"\"\n", " if title: signature += f\"Title: {title}, \"\n", " if author: signature += f\"Author: {author}, \"\n", @@ -470,10 +513,11 @@ " # if tags: signature += f\"Tags: {tags}, \"\n", " if signature: signature = signature[:-2]\n", "\n", - " self.data.append((title, author, date_published, url, tags, text))\n", " \n", - " blocks = self.text_splitter.split(text, signature)\n", - " self.embed_split.extend(blocks)\n", + " self.metadata.append((title, author, date_published, url, tags, text))\n", + " \n", + " blocks = text_splitter.split(text, signature)\n", + " self.embedding_strings.extend(blocks)\n", " \n", " self.total_char_count += len(entry['text'])\n", " self.total_word_count += len(entry['text'].split())\n", @@ -495,13 +539,13 @@ " \n", " embeddings = []\n", " with concurrent.futures.ThreadPoolExecutor() as executor:\n", - " futures = [executor.submit(get_embedding, text) for text in self.embed_split]\n", + " futures = [executor.submit(get_embedding, text) for text in self.embedding_strings]\n", " num_completed = 0\n", " for future in concurrent.futures.as_completed(futures):\n", " embeddings.append(future.result())\n", " num_completed += 1\n", " if num_completed % 50 == 0:\n", - " print(f\"Completed {num_completed}/{len(self.embed_split)}\")\n", + " print(f\"Completed {num_completed}/{len(self.embedding_strings)}\")\n", " self.embeddings = np.vstack(embeddings)\n", " \n", " def save_embeddings(self, path: str):\n", @@ -511,479 +555,90 @@ " self.embeddings = np.load(path)\n", " \n", " def save_class(self, path: str):\n", - " self.text_splitter = None\n", " with open(path, 'wb') as f:\n", " pickle.dump(self, f)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "# List of possible sources:\n", "all_sources = [\"https://aipulse.org\", \"ebook\", \"https://qualiacomputing.com\", \"alignment forum\", \"lesswrong\", \"manual\", \"arxiv\", \"https://deepmindsafetyresearch.medium.com\", \"waitbutwhy.com\", \"GitHub\", \"https://aiimpacts.org\", \"arbital.com\", \"carado.moe\", \"nonarxiv_papers\", \"https://vkrakovna.wordpress.com\", \"https://jsteinhardt.wordpress.com\", \"audio-transcripts\", \"https://intelligence.org\", \"youtube\", \"reports\", \"https://aisafety.camp\", \"curriculum\", \"https://www.yudkowsky.net\", \"distill\"]\n", "\n", - "sources = [\"alignment forum\", #\"lesswrong\", \n", + "# List of sources we are using for the test run:\n", + "custom_sources = [\"alignment forum\", \"lesswrong\", \n", " \"manual\", \"waitbutwhy.com\", \"https://aiimpacts.org\", \"arbital.com\", \"https://intelligence.org\", \"reports\", \"https://aisafety.camp\", \"curriculum\", \"https://www.yudkowsky.net\", \"distill\"]\n", "\n", "\n", - "dataset = Dataset(path=PATH_TO_DATA.resolve(), sources=sources, rate_limit_per_minute=3500, block_min_max_size = [500, 600], fraction_of_articles_to_use=1/1000)\n", + "# dataset = Dataset(jsonl_data_path=PATH_TO_DATA.resolve(), sources=sources, rate_limit_per_minute=3500, block_min_max_size = [500, 600], fraction_of_articles_to_use=1/500)\n", + "dataset = Dataset(jsonl_data_path=PATH_TO_DATA.resolve(), custom_sources=custom_sources, rate_limit_per_minute=3500, min_tokens_per_block=500, max_tokens_per_block=600, fraction_of_articles_to_use=1/2000)\n", "dataset.get_alignment_texts()" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 10, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " I think of ambitious value learning as a proposed solution to the specification problem, which I define as the problem of *defining* the behavior that we would want to see from our AI system. I italicize \"defining\" to emphasize that this is *not* the problem of actually *computing* behavior that we want to see—that’s the full AI safety problem. Here we are allowed to use hopelessly impractical schemes, as long as the resulting definition would allow us to *in theory* compute the behavior that an AI system would take, perhaps with assumptions like infinite computing power or arbitrarily many queries to a human. (Although we do prefer specifications that seem like they could admit an efficient implementation. ) In terms of DeepMind’s classification, we are looking for a design specification that exactly matches the ideal specification. HCH and indirect normativity are examples of attempts at such specifications. We will consider a model in which our AI system is maximizing the expected utility of some *explicitly* represented utility function that can depend on history. (It does not matter materially whether we consider utility functions or reward functions, as long as they can depend on history. ) The utility function may be learned from data, or designed by hand, but it must be an explicit part of the AI that is then maximized. I will not justify this model for now, but simply assume it by fiat and see where it takes us. I’ll note briefly that this model is often justified by the VNM utility theorem and AIXI, and as the natural idealization of reinforcement learning, which aims to maximize the expected sum of rewards, although typically rewards in RL depend only on states. A lot of conceptual arguments, as well as experiences with specification gaming, suggest that we are unlikely to be able to simply think hard and write down a good specification, since even small errors in specifications can lead to bad results. However, machine learning is particularly good at narrowing down on the correct hypothesis among a vast space of possibilities using data, so perhaps we could determine a good specification from some suitably chosen source of data? This leads to the idea of ambitious value learning, where we *learn* an explicit utility function from human behavior for the AI to maximize. This is very related to inverse reinforcement learning (IRL) in the machine learning literature, though not all work on IRL is relevant to ambitious value learning. For example, much work on IRL is aimed at *imitation learning*, which would in the best case allow you to match human performance, but not to exceed it.\n", - " - Title: What is ambitious value learning?, Author: Rohin Shah, Date published: 2018-11-01, URL: https://www.lesswrong.com/posts/5eX8ko7GCxwR5N9mN/what-is-ambitious-value-learning\n", - " aims to maximize the expected sum of rewards, although typically rewards in RL depend only on states. A lot of conceptual arguments, as well as experiences with specification gaming, suggest that we are unlikely to be able to simply think hard and write down a good specification, since even small errors in specifications can lead to bad results. However, machine learning is particularly good at narrowing down on the correct hypothesis among a vast space of possibilities using data, so perhaps we could determine a good specification from some suitably chosen source of data? This leads to the idea of ambitious value learning, where we *learn* an explicit utility function from human behavior for the AI to maximize. This is very related to inverse reinforcement learning (IRL) in the machine learning literature, though not all work on IRL is relevant to ambitious value learning. For example, much work on IRL is aimed at *imitation learning*, which would in the best case allow you to match human performance, but not to exceed it.Ambitious value learning is, well, more ambitious—it aims to learn a utility function that captures \"what humans care about\", so that an AI system that optimizes this utility function more capably can *exceed* human performance, making the world better for humans than they could have done themselves. It may sound like we would have solved the entire AI safety problem if we could do ambitious value learning—surely if we have a good utility function we would be done. Why then do I think of it as a solution to just the specification problem? This is because ambitious value learning by itself would not be enough for safety, except under the assumption of as much compute and data as desired. These are really powerful assumptions—for example, I’m assuming you can get data where you put a human in an arbitrarily complicated simulated environment with fake memories of their life so far and see what they do.\n", - "\n", - " - Attempting to use the utility function to choose actions before it has converged\n", - "\n", - " - Distributional shift causing the learned utility function to become invalid\n", - "\n", - " - Local minima preventing us from learning a good utility function, or from optimizing the learned utility function correctly The next few posts in this sequence will consider the suitability of ambitious value learning as a solution to the specification problem. Most of them will consider whether ambitious value learning is possible in the setting above (infinite compute and data). One post will consider practical issues with the application of IRL to infer a utility function suitable for ambitious value learning, while still assuming that the resulting utility function can be perfectly maximized (which is equivalent to assuming infinite compute and a perfect model of the environment *after* IRL has run).\n", - " - Title: What is ambitious value learning?, Author: Rohin Shah, Date published: 2018-11-01, URL: https://www.lesswrong.com/posts/5eX8ko7GCxwR5N9mN/what-is-ambitious-value-learning\n", - " Contents - Key Definitions - A Sketch of LFAI - Appendix: More Conceptual Clarifications on LFAI - Applicability of Law to AI Systems - Predicting Legality *This post is written in my personal capacity, and does not necessarily represent the views of OpenAI or any other organization. Cross-posted to the Effective Altruism Forum.\n", - "\n", - " This sequence of posts will argue that working to ensure that AI systems follow laws is a worthwhile way to improve the long-term future of AI.\n", - "\n", - " The structure of this sequence will be as follows:\n", - "\n", - " - First, in this post, I will define some key terms and sketch what an ideal law-following AI (\"LFAI\") system might look like.\n", - "\n", - " - In the next few posts, I will explain why law-following might not emerge by default given the existing constellation of alignment approaches, financial objectives, and legal constraints, and explain why this is troubling.\n", - "\n", - " - Finally, I will propose some policy and technical routes to ameliorating these problems.\n", - "\n", - " If the vision here excites you, and you would like to get funding to work on it, get in touch. I may be excited to recommend grants for people working on this, as long as it does not distract them from working on more important alignment issues.\n", - "\n", - " \n", - "\n", - " *Image by OpenAI’s DALL·E.\n", - "\n", - " ## Key Definitions\n", - "\n", - " A **law-following AI** , or **LFAI** , is an AI system that is designed to rigorously comply with some defined set of human-originating rules (\"laws\"),[2] using legal interpretative techniques,[3] under the assumption that those laws apply to the AI in the same way that they would to a human. By \"intrinsically motivated,\" I mean that the AI is motivated to obey those rules regardless of whether (a) its human principal wants it to obey the law,[4] or (b) disobeying the law would be instrumentally valuable. [5] (The Appendix to this post explores some possible conceptual issues with this definition of LFAI.\n", - "\n", - " I will compare LFAI with **intent-aligned AI**.\n", - " - Title: Law-Following AI 1: Sequence Introduction and Structure, Author: Cullen_OKeefe, Date published: 2022-04-27, URL: https://www.lesswrong.com/posts/NrtbF3JHFnqBCztXC/law-following-ai-1-sequence-introduction-and-structure\n", - " - Jan Leike et al. define the \"agent alignment problem\" as \"How can we create agents that behave in accordance with the user’s intentions\"?\n", - "\n", - " - Amanda Askell et al. define \"alignment\" as \"the degree of overlap between the way two agents rank different outcomes\".\n", - "\n", - " - Paul Christiano defines \"AI alignment\" as \"*A* is trying to do what *H* wants it to do\".\n", - "\n", - " - Richard Ngo endorses Christiano’s definition.\n", - "\n", - " Iason Gabriel does not directly define \"intent alignment,\" but provides a taxonomy wherein an AI agent can be aligned with:\n", - "\n", - " - \"Instructions: the agent does what I instruct it to do\".\n", - "\n", - " - \"Expressed intentions: the agent does what I intend it to do\".\n", - "\n", - " - \"Revealed preferences: the agent does what my behaviour reveals I prefer\".\n", - "\n", - " - \"Informed preferences or desires: the agent does what I would want it to do if I were rational and informed\".\n", - "\n", - " - \"Interest or well-being: the agent does what is in my interest, or what is best for me, objectively speaking\".\n", - "\n", - " - \"Values: the agent does what it morally ought to do, as defined by the individual or society\".\n", - "\n", - " All but (6) concern the relationship between *H* and *A*. It would therefore seem appropriate to describe them as types of intent alignment.\n", - "\n", - " Alignment with some broader or more complete set of values—such as type (6) in Gabriel’s taxonomy, Coherent Extrapolated Volition, or what Ngo calls \"maximalist\" or \"ambitious\" alignment—is perhaps desirable or even necessary, but seems harder than working on intent alignment. [6] Much current alignment work therefore focuses on intent alignment.\n", - "\n", - " We can see that, on its face, intent alignment does not entail law-following.\n", - "\n", - " - Bad in expectation for the long-term future.\n", - "\n", - " - Easier to bridge than the gap between intent alignment and deeper alignment with moral truth.\n", - "\n", - " - Therefore worth addressing.\n", - "\n", - " To clarify, this sequence does **not** claim that LFAI can replace intent alignment.\n", - "\n", - " ## A Sketch of LFAI\n", - "\n", - " What might an LFAI system look like? I’m not a computer scientist, but here is roughly what I have in mind.\n", - " - Title: Law-Following AI 1: Sequence Introduction and Structure, Author: Cullen_OKeefe, Date published: 2022-04-27, URL: https://www.lesswrong.com/posts/NrtbF3JHFnqBCztXC/law-following-ai-1-sequence-introduction-and-structure\n", - " If *A* is an LFAI, then *A*’s evaluation of the legality of an action will sometimes trump *A*’s evaluation of an action in light of its benefit to *H*. In LFAI, as in a legally scrupulous human, legality constrains how an agent can advance their principal’s interests. For example, a human mover may be instructed to efficiently move a box for her principal, but may not unnecessarily destroy others’ property in doing so. Similarly, an LFAI moving a box normally would not knock over a vase in its path, because doing so would violate the legal rights of the vase-owner.\n", - "\n", - " Above, I preliminarily defined LFAI as \"rigorously comply[ing]\" with some set of laws. Obviously this needs a bit more elaboration. We probably don’t want to define this as *minimizing* legal noncompliance, since this would make the system extremely risk-averse to the point of being useless. More likely, one would attempt to weight legal downside risks heavily in the agent’s objective function,[8] such that it would keep legal risk to an acceptable level.\n", - "\n", - " It is worth noting that LFAI is ideally not merely attempting to reduce its expected legal liability *in fact*. As will be explored later, a sufficiently smart agent could probably reduce its expected legal liability merely by hiding its knowledge/​intentions/​actions or corrupting a legal proceeding. An LFAI, by contrast, is attempting to obey the law in an idealized sense, even if it is unlikely to actually face legal consequences.\n", - "\n", - " An LFAI system does not need to store all knowledge regarding the set of laws that it is trained to follow. More likely, the practical way to create such a system would be to make the system capable of recognizing when it faces sufficient legal uncertainty,[10] then seeking evaluation from a legal expert system (\"Counselor\").\n", - "\n", - " The Counselor could be a human lawyer, but in the long-run is probably most robust and efficient if (at least partially) automated. The Counselor would then render advice on the pure basis of idealized legality: the probability and expected legal downsides that would result from an idealized legal dispute regarding the action if everyone knew all the relevant facts.\n", - " - Title: Law-Following AI 1: Sequence Introduction and Structure, Author: Cullen_OKeefe, Date published: 2022-04-27, URL: https://www.lesswrong.com/posts/NrtbF3JHFnqBCztXC/law-following-ai-1-sequence-introduction-and-structure\n", - " Thus pseudocode for an LFAI who wants to take an action *X* to benefit *H* might be:\n", - "\n", - " - If *X* is clearly illegal:\n", - "\n", - " - don’t do *X*.\n", - "\n", - " \n", - "\n", - " - Elseif *X* is maybe-illegal:\n", - "\n", - " - Give Counselor all relevant information about *X* in an unbiased way; then\n", - "\n", - " - Get Counselor’s opinion on expected legal consequences from *X*; then\n", - "\n", - " - Weigh expected legal consequences against benefit to *H* from *X*; then\n", - "\n", - " - Decide whether to do *X* given those weightings.\n", - "\n", - " \n", - "\n", - " - Else:\n", - "\n", - " - do *X*.\n", - "\n", - " \n", - "\n", - " Note that this pseudocode may resemble the decisionmaking process of *A* if *H* wants *A* to obey the law. Thus, one route to giving an intent-aligned AI the motivation to obey the law may be stipulating to *A* that *H* wants *A* to obey the law.\n", - "\n", - " With this picture in mind, it seems like, to make LFAI a reality, progress on the following open problems (non-exhaustively) would be useful:\n", - "\n", - " - Reliably stipulating low-following conditions to AI systems’ objectives.\n", - "\n", - " - Resolving any disagreement between law-following and a principal’s instructions appropriately.\n", - "\n", - " \n", - "\n", - " - Getting AI agents to recognize when they face legal uncertainty (especially in a way that does not incentivize ignorance of the law).\n", - "\n", - " - This seems similar to the intent alignment problem of getting agents to recognize when they need further information from principals, as in corrigibility work.\n", - "\n", - " \n", - "\n", - " - Eliciting, in natural language, AI systems’ honest description of its knowledge and desired actions.\n", - "\n", - " - As noted above, this seems likely to run into problems related to ELK generally.\n", - "\n", - " \n", - "\n", - " - Mapping legal concepts of mental states (e.g., intent, knowledge) to features of AI systems.\n", - "\n", - " - This seems related to interpretability and explainability work.\n", - "\n", - " \n", - "\n", - " - Building Counselor functions.\n", - "\n", - " - Automating the process of legal research given a natural language description of an agent’s proposed actions and mental state.\n", - " - Title: Law-Following AI 1: Sequence Introduction and Structure, Author: Cullen_OKeefe, Date published: 2022-04-27, URL: https://www.lesswrong.com/posts/NrtbF3JHFnqBCztXC/law-following-ai-1-sequence-introduction-and-structure\n", - " - Simulating idealized and fair substantive legal disputes.\n", - "\n", - " - This seems related to Debate.\n", - "\n", - " \n", - "\n", - " \n", - "\n", - " ## Appendix: More Conceptual Clarifications on LFAI\n", - "\n", - " This Appendix provides some additional clarification on the definition of LFAI given above.\n", - "\n", - " ### Applicability of Law to AI Systems\n", - "\n", - " One might worry that the law often regulates physical behavior in a way that is not obviously applicable to all AI systems. For example, physical contact with another is an element of the tort of battery. [14] However, this may be less of a problem than initially appears: courts have been able to reason through whether to apply laws originating in meatspace to computational and cyberspace conduct. [15] Whether such analogies are *properly* applied is indeed highly debatable,[16] but the fact that such analogizing is conceptually possible reduces the force of this objection. Furthermore, if some laws are simply inapplicable to non-embodied actors, this is not a problem for the conceptual coherence of LFAI as a whole: an LFAI can simply ignore those laws,[17] and we can design laws specifically with computational content.\n", - "\n", - " Perhaps a more fundamental problem is that the law frequently depends on *mental states* that are not straightforwardly applicable to AI systems. For example, the legality of an action may depend on whether the actor *intended* some harmful outcome. Thus, much of the value of LFAI depends on whether we can map human understandings of moral culpability to AI systems.\n", - "\n", - " To me, however, this seems like an argument in favorof working on LFAI. Regardless of whether LFAI as such is valuable, if we expect increasingly autonomous AI systems to take increasingly impactful actions, we would probably like to understand how their objective functions (analogous to human *motives*) and world-model (analogous to human *knowledge*) map to their actions and the effects thereof. This is for the same reasons that we care about human motives and knowledge: when evaluating the alignment of agents, it is useful to know whether an agent intended to cause some harm, or knew that such a harm would ensue, etc. LFAI depends on progress on this, but is also potentially a useful toy problem for interpretability and related work in ML.\n", - " - Title: Law-Following AI 1: Sequence Introduction and Structure, Author: Cullen_OKeefe, Date published: 2022-04-27, URL: https://www.lesswrong.com/posts/NrtbF3JHFnqBCztXC/law-following-ai-1-sequence-introduction-and-structure\n", - " ### Predicting Legality\n", - "\n", - " Legal compliance is also a function of both law and facts, and responsibility for definitive determinations of law and facts is split between judges and juries. Law often invokes standards like \"reasonableness\" that are definitively assessed only ex post, in the context of a particular dispute. The definitive legality of an action may therefore turn on an actual adjudication of the dispute. This is of course costly, which is why I suspect we would want an LFAI to act on its best estimate of what such an adjudication would yield (after asking a Counselor), rather than wait for such adjudication to take place.\n", - "\n", - " It is also worth distinguishing between whether an *actual court of law* would rule that an AI’s behavior violated some law and whether a *simulated and fair legal dispute resolution process* (possibly including, for example, a bespoke arbitral panel) would conclude that the behavior violated the law. The latter may be more convenient for working on LFAI for a number of reasons, including that it can ignore or stipulate away some of the peculiarities of adjudicating disputes in which an AI system is a \"party\".\n", - "\n", - " \n", - "\n", - " - For early, informal discussion on this topic, see Michael St. Jules, *What are the challenges and problems with programming law-breaking constraints into AGI? *, **Effective Altruism Forum** (Feb. 2, 2020), https://​​forum. effectivealtruism.org/​​posts/​​qKXLpe7FNCdok3uvY/​​what-are-the-challenges-and-problems-with-programming-law [https://​​perma. cc/​​HJ4Y-XSSE] and accompanying comments.\n", - "\n", - " - Whether such rules are actually encoded into legislation is not particularly important. Virtually all legal rules not part of public law can be made \"legal\" with regards to particular parties as part of a contract, for example. In any case, the heart of LFAI is being bound to follow rules, and interpreting those rules leveraging the rich body of useful rule-interpretation metarules from law.\n", - " - Title: Law-Following AI 1: Sequence Introduction and Structure, Author: Cullen_OKeefe, Date published: 2022-04-27, URL: https://www.lesswrong.com/posts/NrtbF3JHFnqBCztXC/law-following-ai-1-sequence-introduction-and-structure\n", - " - This is important because one of the core functions of law is to provide metarules regarding the interpretation of rules, guided by certain normative values (e.g., fairness, predictability, consistency). Indeed, rules of legal interpretation aim to solve many problems relevant to AI interpretation of instructions. *Cf. * Dylan Hadfield-Menell & Gillian Hadfield, *Incomplete Contracting and AI Alignment* (2018) (preprint), https://​​arxiv.org/​​abs/​​1804.04268.\n", - "\n", - " - That is, the AI is not law-following *just because* the principal wants the AI to follow the law. Indeed, LFAI should disobey orders that would require it to behave illegally.\n", - "\n", - " - That is, the AI is not law-following *just because* it is instrumentally valuable to it (because, e.g., being caught breaking the law would cause the AI to be turned off).\n", - "\n", - " - As Ngo says, \"My opinion is that defining alignment in maximalist terms is unhelpful, because it bundles together technical, ethical and political problems. While it may be the case that we need to make progress on all of these, assumptions about the latter two can significantly reduce clarity about technical issues\".\n", - "\n", - " - *Cf. , e.g.*, Dario Amodei et al. , Concrete Problems in AI Safety 4 (2016), https://​​arxiv.org/​​pdf/​​1606.06565. pdf.\n", - "\n", - " - I don’t here offer an opinion on what training regime would yield such an outcome—my hope is to get someone to answer that for me!\n", - "\n", - " - This approach may work particularly well when combined with insurance requirements for people deploying AI systems.\n", - "\n", - " - In the same way that an intent-aligned AI will sometimes ask for clarifications from a human principal. *See* Christiano.\n", - "\n", - " - Note that there are ELK-style problems with this approach. If an AI is asking for legal advice and wants to minimize the negative signal it gets from the Counselor, it may hide certain relevant information (e.g., its true state of knowledge or its true intentions) from the Counselor. A good solution, as discussed, could be to simulate an idealized adjudication of the issue if all the parties knew all the relevant facts and had equal legal firepower. But incentivizing the LFAI to tell the Counselor its true knowledge/​intentions is an ELK problem.\n", - " - Title: Law-Following AI 1: Sequence Introduction and Structure, Author: Cullen_OKeefe, Date published: 2022-04-27, URL: https://www.lesswrong.com/posts/NrtbF3JHFnqBCztXC/law-following-ai-1-sequence-introduction-and-structure\n", - "In the limit, the Counselor need not strictly be a distinct agent from the LFAI: an LFAI system may have Counselor capabilities and run this \"consultation\" process internally. Nevertheless, it is illustratively useful to imagine a separation of the LFAI and the Counselor.\n", - "\n", - " - This would be *idealized* so that details not ultimately relevant to the substantive legality of the action (e.g., jurisdiction, AI personhood, other procedural matters, asymmetries in legal firepower) can be ignored. See the final footnote of this piece for further discussion.\n", - "\n", - " - See the Appendix for more discussion on this point.\n", - "\n", - " - *See Battery*, **Wex** , https://​​www. law. cornell.edu/​​wex/​​battery (last accessed Sept. 3, 2021).\n", - "\n", - " - *See, e.g.*, Intel Corp. v. Hamidi, 71 P.3d 296, 304–08 (Cal. 2003) (applying trespass to chattels to unauthorized electronic computer access); MAI Sys. Corp. v. Peak Computer, Inc., 991 F.2d 511, 518–19 (9th Cir. 1993) (storing data in RAM sufficient to create a \"copy\" for copyright purposes, despite the fact that a \"copy\" must be \"fixed in a tangible medium\"); *cf. * United States v. Jones, 565 U.S. 400, 406 n.3 (2012) (analogizing GPS tracking to in-person surveillance for Fourth Amendment purposes).\n", - "\n", - " - *See, e.g.*, Jonathan H. Blavin & I. Glenn Cohen, *Gore, Gibson, and Goldsmith: The Evolution of Internet Metaphors in Law and Commentary*, 16 **Harv. J.L. & Tech. ** 265 (2002).\n", - "\n", - " - However, the case for working on LFAI certainly diminishes with the number of applicable laws.\n", - "\n", - " - This raises further issues, including the possibility of self-reference. For example, an LFAI or Counselor asymmetrically deployed by one litigant may be able to persuade a judge or jury of its position, even if it’s not the best outcome. To avoid this, such simulations should assume that judges and juries are fully apprised of all relevant facts (i.e., neither the LFAI nor Counselor can obscure relevant evidence) and if deployed in the simulated proceeding are symmetrically available to both sides.\n", - " - Title: Law-Following AI 1: Sequence Introduction and Structure, Author: Cullen_OKeefe, Date published: 2022-04-27, URL: https://www.lesswrong.com/posts/NrtbF3JHFnqBCztXC/law-following-ai-1-sequence-introduction-and-structure\n", - " *Overall summary post here.\n", - "\n", - " When designing a low-impact agent A, one typically wants them to maximise R_0 - \\rho, where R_0 is some positive reward function we want A to increase (ie the actual goal) and \\rho is some impact measure or impact penalty. Keeping \\rho low is supposed to ensure that A doesn’t make a big difference to the world.\n", - "\n", - " I’ve shown that some impact measures may have issues with subagents.\n", - "\n", - " - Can A build SA, an \"R_0 -\\rho\"-maximising subagent, without triggering a high \\rho?\n", - "\n", - " - Does SA allow A to cheat the impact measure and get higher reward?\n", - "\n", - " I’ll consider three impact measures: the twenty billion questions measure from my old low-impacts paper, the relative reachability measure from Krakovna et al’s paper, and the attainable utility measure from Turner et al’s paper.\n", - "\n", - " Then the answers to those two questions for those three impact measures are generally[1]:\n", - "\n", - " \\begin{array}{|c||c|c|} \\hline \\rho & \\textrm{Possible to build } SA\\textrm{? } & \\textrm{Incentive to build } SA\\textrm{?\n", - "\n", - " ## Building the subagent\n", - "\n", - " The twenty billion questions impact measure does not prevent the agent from building a subagent within its box. That’s because that measure is keyed to changes in the world, not to the power level of the agent. Until there are changes in the world, it will not trigger.\n", - "\n", - " The relative reachability impact measure similarly won’t penalise the construction of a subagent (as long as the \"states\" it uses to define reachability don’t include specific information about the agent and any subagents). The reasoning is similar: the penalty triggers when an \"irreversible\" state is reached, not when the agent becomes powerful.\n", - "\n", - " Building the subagent for the attainable utility impact measure is harder, since this typically increases the power of the agent. But the question is not \"will any subagent trigger the impact measure\"? , but \"can the agent build a subagent *in some way* that doesn’t trigger the impact measure\"? The example here shows that this is possible in some cases. Given that we don’t have any impossibility results here, we can expect that a superintelligent agent will likely find a way of doing it.\n", - " - Title: Building and using the subagent, Author: Stuart_Armstrong, Date published: 2020-02-12, URL: https://www.lesswrong.com/posts/fqJmZBG5xmvvd8WRi/building-and-using-the-subagent\n", - " subagent for the attainable utility impact measure is harder, since this typically increases the power of the agent. But the question is not \"will any subagent trigger the impact measure\"? , but \"can the agent build a subagent *in some way* that doesn’t trigger the impact measure\"? The example here shows that this is possible in some cases. Given that we don’t have any impossibility results here, we can expect that a superintelligent agent will likely find a way of doing it. ## Using the subagent\n", - "\n", - " Does having the subagent help to evade the impact measure (and hence increase R_0 more)? For twenty billion questions, the answer is generally no[2]. Having SA as an independent agent is no different from having SA as a direct extension of A. If SA changes anything about the various variables in \\rho, that triggers the impact penalty just as much as if A had done so.\n", - "\n", - " But the situation is different for the attainable utility penalty. That’s because that penalty is *indexed* to the agent. So, for A, the impact penalty is roughly \"ensure that **you** don’t get too much power\". But for SA, the impact penalty is \"ensure that **agent** \\bf{A} doesn’t get too much power\". Given that A is actively cooperating with SA, that second penalty is much easier to satisfy, while still maximising R_0: see here for how SA can then become an essentially un-restricted R_0-maximiser.\n", - "\n", - " For relative reachability, it depends on the baseline. For the inaction baseline, the \\rho is entirely defined in terms of a distance metric between the current state s_t and a given state s_t' that is independent of the agent. So, like the twenty billion questions, subagents don’t allow the agent to evade the impact penalty.\n", - "\n", - " But if the stepwise inaction baseline is used, then s_t' is defined by \"what would have happened if A had done nothing the previous turn (and the future n turns)\". This is also an indexed penalty, so SA can easily maximise R_0 while A does nothing, thus ensuring s_t=s_t' and the penalty vanishes.\n", - "\n", - " \n", - "\n", - " - Since all impact measures can apply to many different setups, we can’t say anything fully definitive about every situation that uses a particular impact measure.\n", - "\n", - " - Technically, A might like to construct an SA far away, beyond the \"reach\" of \\rho. But that’s because A might like to move *itself* beyond the reach of \\rho - the subagent is just an indirect way of doing so.\n", - " - Title: Building and using the subagent, Author: Stuart_Armstrong, Date published: 2020-02-12, URL: https://www.lesswrong.com/posts/fqJmZBG5xmvvd8WRi/building-and-using-the-subagent\n", - " Last year Stuart Armstrong announced a contest to come up with the best questions to ask an Oracle AI. Wei Dai wrote, > Submission. For the counterfactual Oracle, ask the Oracle to predict the n best posts on AF during some future time period (counterfactually if we didn’t see the Oracle’s answer). He later related his answer to Paul Christiano’s posts on Human-in-the-counterfactual-loop and Elaborations on apprenticeship learning. Here I’m interested in *concrete* things that can be expected to go wrong in the near future if we gave GPT-N this task. To provide a specific example, suppose we provided the prompt, *> This is the first post in an Alignment Forum sequence explaining the approaches both MIRI and OpenAI staff believe are the most promising means of auditing the cognition of very complex machine learning models. *If by assumption, GPT-N is at least as good as a human expert team at generating blog posts, we could presumably expect this GPT-N to produce a very high quality post explaining how to inspect machine learning models. We would therefore have a way of to automate alignment research at a high level.\n", - "\n", - " - How large would GPT-N need to be before it started producing answers comparable to a human expert team, and\n", - "\n", - " - Given the size of the model, what high-level incentives should we expect to guide the training of the model?\n", - "\n", - " - Is there a clear and unambiguous danger that the model would be manipulative? If so, why?\n", - "\n", - " - Is the threat model more that we don’t know what we don’t know, or that we have a specific reason to believe the model would be manipulative in a particular direction?\n", - " - Title: [Question] What specific dangers arise when asking GPT-N to write an Alignment Forum post?, Author: Matthew Barnett, Date published: 2020-07-28, URL: https://www.lesswrong.com/posts/Et2pWrj4nWfdNAawh/what-specific-dangers-arise-when-asking-gpt-n-to-write-an\n", - " New paper: “Risks from learned optimization”\n", - "\n", - " Evan Hubinger, Chris van Merwijk, Vladimir Mikulik, Joar Skalse, and Scott Garrabrant have a new paper out: “Risks from learned optimization in advanced machine learning systems”. The paper’s abstract: We analyze the type of learned optimization that occurs when a learned model (such as a neural network) is itself an optimizer—a situation we refer to as mesa-optimization, a neologism we introduce in this paper. We believe that the possibility of mesa-optimization raises two important questions for the safety and transparency of advanced machine learning systems. First, under what circumstances will learned models be optimizers, including when they should not be? Second, when a learned model is an optimizer, what will its objective be—how will it differ from the loss function it was trained under—and how can it be aligned? In this paper, we provide an in-depth analysis of these two primary questions and provide an overview of topics for future research. The critical distinction presented in the paper is between what an AI system is optimized to do (its base objective) and what it actually ends up optimizing for (its mesa-objective), if it optimizes for anything at all. The authors are interested in when ML models will end up optimizing for something, as well as how the objective an ML model ends up optimizing for compares to the objective it was selected to achieve. The distinction between the objective a system is selected to achieve and the objective it actually optimizes for isn’t new. Eliezer Yudkowsky has previously raised similar concerns in his discussion of optimization daemons, and Paul Christiano has discussed such concerns in “What failure looks like”. The paper’s contents have also been released this week as a sequence on the AI Alignment Forum, cross-posted to LessWrong. As the authors note there: We believe that this sequence presents the most thorough analysis of these questions that has been conducted to date. In particular, we plan to present not only an introduction to the basic concerns surrounding mesa-optimizers, but also an analysis of the particular aspects of an AI system that we believe are likely to make the problems related to mesa-optimization relatively easier or harder to solve. By providing a framework for understanding the degree to which different AI systems are likely to be robust to misaligned mesa-optimization, we hope to start a discussion about the best ways of structuring machine learning systems to solve these problems.\n", - " - Title: New paper: “Risks from learned optimization”, Author: Rob Bensinger, Date published: Fri, 07 Jun 2019, URL: https://intelligence.org/2019/06/07/new-paper-learned-optimization/\n", - " to as mesa-optimization, a neologism we introduce in this paper. We believe that the possibility of mesa-optimization raises two important questions for the safety and transparency of advanced machine learning systems. First, under what circumstances will learned models be optimizers, including when they should not be? Second, when a learned model is an optimizer, what will its objective be—how will it differ from the loss function it was trained under—and how can it be aligned? In this paper, we provide an in-depth analysis of these two primary questions and provide an overview of topics for future research. The critical distinction presented in the paper is between what an AI system is optimized to do (its base objective) and what it actually ends up optimizing for (its mesa-objective), if it optimizes for anything at all. The authors are interested in when ML models will end up optimizing for something, as well as how the objective an ML model ends up optimizing for compares to the objective it was selected to achieve. The distinction between the objective a system is selected to achieve and the objective it actually optimizes for isn’t new. Eliezer Yudkowsky has previously raised similar concerns in his discussion of optimization daemons, and Paul Christiano has discussed such concerns in “What failure looks like”. The paper’s contents have also been released this week as a sequence on the AI Alignment Forum, cross-posted to LessWrong. As the authors note there: We believe that this sequence presents the most thorough analysis of these questions that has been conducted to date. In particular, we plan to present not only an introduction to the basic concerns surrounding mesa-optimizers, but also an analysis of the particular aspects of an AI system that we believe are likely to make the problems related to mesa-optimization relatively easier or harder to solve. By providing a framework for understanding the degree to which different AI systems are likely to be robust to misaligned mesa-optimization, we hope to start a discussion about the best ways of structuring machine learning systems to solve these problems.Furthermore, in the fourth post we will provide what we think is the most detailed analysis yet of a problem we refer as deceptive alignment which we posit may present one of the largest—though not necessarily insurmountable—current obstacles to producing safe advanced machine learning systems using techniques similar to modern machine learning.\n", - "\n", - " \n", - "\n", - " Sign up to get updates on new MIRI technical results Get notified every time a new technical paper is published.\n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "\n", - " ×\n", - "\n", - " The post New paper: “Risks from learned optimization” appeared first on Machine Intelligence Research Institute.\n", - " - Title: New paper: “Risks from learned optimization”, Author: Rob Bensinger, Date published: Fri, 07 Jun 2019, URL: https://intelligence.org/2019/06/07/new-paper-learned-optimization/\n", - " The Rocket Alignment Problem\n", - "\n", - " The following is a fictional dialogue building off of AI Alignment: Why It’s Hard, and Where to Start.\n", - "\n", - " (Somewhere in a not-very-near neighboring world, where science took a very different course…)   ALFONSO:  Hello, Beth. I’ve noticed a lot of speculations lately about “spaceplanes” being used to attack cities, or possibly becoming infused with malevolent spirits that inhabit the celestial realms so that they turn on their own engineers. I’m rather skeptical of these speculations. Indeed, I’m a bit skeptical that airplanes will be able to even rise as high as stratospheric weather balloons anytime in the next century. But I understand that your institute wants to address the potential problem of malevolent or dangerous spaceplanes, and that you think this is an important present-day cause. BETH:  That’s… really not how we at the Mathematics of Intentional Rocketry Institute would phrase things. The problem of malevolent celestial spirits is what all the news articles are focusing on, but we think the real problem is something entirely different. We’re worried that there’s a difficult, theoretically challenging problem which modern-day rocket punditry is mostly overlooking. We’re worried that if you aim a rocket at where the Moon is in the sky, and press the launch button, the rocket may not actually end up at the Moon. ALFONSO:  I understand that it’s very important to design fins that can stabilize a spaceplane’s flight in heavy winds. That’s important spaceplane safety research and someone needs to do it. But if you were working on that sort of safety research, I’d expect you to be collaborating tightly with modern airplane engineers to test out your fin designs, to demonstrate that they are actually useful. BETH:  Aerodynamic designs are important features of any safe rocket, and we’re quite glad that rocket scientists are working on these problems and taking safety seriously. That’s not the sort of problem that we at MIRI focus on, though. ALFONSO:  What’s the concern, then? Do you fear that spaceplanes may be developed by ill-intentioned people? BETH:  That’s not the failure mode we’re worried about right now. We’re more worried that right now, nobody can tell you how to point your rocket’s nose such that it goes to the moon, nor indeed any prespecified celestial destination.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "Whether Google or the US Government or North Korea is the one to launch the rocket won’t make a pragmatic difference to the probability of a successful Moon landing from our perspective, because right now nobody knows how to aim any kind of rocket anywhere.\n", - "\n", - " ALFONSO:  I’m not sure I understand. BETH:  We’re worried that even if you aim a rocket at the Moon, such that the nose of the rocket is clearly lined up with the Moon in the sky, the rocket won’t go to the Moon. We’re not sure what a realistic path from the Earth to the moon looks like, but we suspect it might not be a very straight path, and it may not involve pointing the nose of the rocket at the moon at all. We think the most important thing to do next is to advance our understanding of rocket trajectories until we have a better, deeper understanding of what we’ve started calling the “rocket alignment problem”. There are other safety problems, but this rocket alignment problem will probably take the most total time to work on, so it’s the most urgent. ALFONSO:  Hmm, that sounds like a bold claim to me. Do you have a reason to think that there are invisible barriers between here and the moon that the spaceplane might hit? Are you saying that it might get very very windy between here and the moon, more so than on Earth? Both eventualities could be worth preparing for, I suppose, but neither seem likely. BETH:  We don’t think it’s particularly likely that there are invisible barriers, no. And we don’t think it’s going to be especially windy in the celestial reaches — quite the opposite, in fact. The problem is just that we don’t yet know how to plot any trajectory that a vehicle could realistically take to get from Earth to the moon. ALFONSO:  Of course we can’t plot an actual trajectory; wind and weather are too unpredictable. But your claim still seems too strong to me. Just aim the spaceplane at the moon, go up, and have the pilot adjust as necessary. Why wouldn’t that work? Can you prove that a spaceplane aimed at the moon won’t go there? BETH:  We don’t think we can prove anything of that sort, no. Part of the problem is that realistic calculations are extremely hard to do in this area, after you take into account all the atmospheric friction and the movements of other celestial bodies and such. We’ve been trying to solve some drastically simplified problems in this area, on the order of assuming that there is no atmosphere and that all rockets move in perfectly straight lines.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "Even those unrealistic calculations strongly suggest that, in the much more complicated real world, just pointing your rocket’s nose at the Moon also won’t make your rocket end up at the Moon. I mean, the fact that the real world is more complicated doesn’t exactly make it any easier to get to the Moon. ALFONSO:  Okay, let me take a look at this “understanding” work you say you’re doing… Huh. Based on what I’ve read about the math you’re trying to do, I can’t say I understand what it has to do with the Moon. Shouldn’t helping spaceplane pilots exactly target the Moon involve looking through lunar telescopes and studying exactly what the Moon looks like, so that the spaceplane pilots can identify particular features of the landscape to land on? BETH:  We think our present stage of understanding is much too crude for a detailed Moon map to be our next research target. We haven’t yet advanced to the point of targeting one crater or another for our landing. We can’t target anything at this point. It’s more along the lines of “figure out how to talk mathematically about curved rocket trajectories, instead of rockets that move in straight lines”. Not even realistically curved trajectories, right now, we’re just trying to get past straight lines at all – ALFONSO:  But planes on Earth move in curved lines all the time, because the Earth itself is curved. It seems reasonable to expect that future spaceplanes will also have the capability to move in curved lines. If your worry is that spaceplanes will only move in straight lines and miss the Moon, and you want to advise rocket engineers to build rockets that move in curved lines, well, that doesn’t seem to me like a great use of anyone’s time. BETH:  You’re trying to draw much too direct of a line between the math we’re working on right now, and actual rocket designs that might exist in the future. It’s not that current rocket ideas are almost right, and we just need to solve one or two more problems to make them work. The conceptual distance that separates anyone from solving the rocket alignment problem is much greater than that. Right now everyone is confused about rocket trajectories, and we’re trying to become less confused. That’s what we need to do next, not run out and advise rocket engineers to build their rockets the way that our current math papers are talking about. Not until we stop being confused about extremely basic questions like why the Earth doesn’t fall into the Sun.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "ALFONSO:  I don’t think the Earth is going to collide with the Sun anytime soon. The Sun has been steadily circling the Earth for a long time now. BETH:  I’m not saying that our goal is to address the risk of the Earth falling into the Sun. What I’m trying to say is that if humanity’s present knowledge can’t answer questions like “Why doesn’t the Earth fall into the Sun? ” then we don’t know very much about celestial mechanics and we won’t be able to aim a rocket through the celestial reaches in a way that lands softly on the Moon. As an example of work we’re presently doing that’s aimed at improving our understanding, there’s what we call the “tiling positions” problem. The tiling positions problem is how to fire a cannonball from a cannon in such a way that the cannonball circumnavigates the earth over and over again, “tiling” its initial coordinates like repeating tiles on a tessellated floor – ALFONSO:  I read a little bit about your work on that topic. I have to say, it’s hard for me to see what firing things from cannons has to do with getting to the Moon. Frankly, it sounds an awful lot like Good Old-Fashioned Space Travel, which everyone knows doesn’t work. Maybe Jules Verne thought it was possible to travel around the earth by firing capsules out of cannons, but the modern study of high-altitude planes has completely abandoned the notion of firing things out of cannons. The fact that you go around talking about firing things out of cannons suggests to me that you haven’t kept up with all the innovations in airplane design over the last century, and that your spaceplane designs will be completely unrealistic. BETH:  We know that rockets will not actually be fired out of cannons. We really, really know that. We’re intimately familiar with the reasons why nothing fired out of a modern cannon is ever going to reach escape velocity. I’ve previously written several sequences of articles in which I describe why cannon-based space travel doesn’t work. ALFONSO:  But your current work is all about firing something out a cannon in such a way that it circles the earth over and over. What could that have to do with any realistic advice that you could give to a spaceplane pilot about how to travel to the Moon? BETH:  Again, you’re trying to draw much too straight a line between the math we’re doing right now, and direct advice to future rocket engineers.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "We think that if we could find an angle and firing speed such that an ideal cannon, firing an ideal cannonball at that speed, on a perfectly spherical Earth with no atmosphere, would lead to that cannonball entering what we would call a “stable orbit” without hitting the ground, then… we might have understood something really fundamental and important about celestial mechanics. Or maybe not! It’s hard to know in advance which questions are important and which research avenues will pan out. All you can do is figure out the next tractable-looking problem that confuses you, and try to come up with a solution, and hope that you’ll be less confused after that. ALFONSO:  You’re talking about the cannonball hitting the ground as a problem, and how you want to avoid that and just have the cannonball keep going forever, right? But real spaceplanes aren’t going to be aimed at the ground in the first place, and lots of regular airplanes manage to not hit the ground. It seems to me that this “being fired out of a cannon and hitting the ground” scenario that you’re trying to avoid in this “tiling positions problem” of yours just isn’t a failure mode that real spaceplane designers would need to worry about. BETH:  We are not worried about real rockets being fired out of cannons and hitting the ground. That is not why we’re working on the tiling positions problem. In a way, you’re being far too optimistic about how much of rocket alignment theory is already solved! We’re not so close to understanding how to aim rockets that the kind of designs people are talking about now would work if only we solved a particular set of remaining difficulties like not firing the rocket into the ground. You need to go more meta on understanding the kind of progress we’re trying to make. We’re working on the tiling positions problem because we think that being able to fire a cannonball at a certain instantaneous velocity such that it enters a stable orbit… is the sort of problem that somebody who could really actually launch a rocket through space and have it move in a particular curve that really actually ended with softly landing on the Moon would be able to solve easily. So the fact that we can’t solve it is alarming. If we can figure out how to solve this much simpler, much more crisply stated “tiling positions problem” with imaginary cannonballs on a perfectly spherical earth with no atmosphere, which is a lot easier to analyze than a Moon launch, we might thereby take one more incremental step towards eventually becoming the sort of people who could plot out a Moon launch.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "ALFONSO:  If you don’t think that Jules-Verne-style space cannons are the wave of the future, I don’t understand why you keep talking about cannons in particular. BETH:  Because there’s a lot of sophisticated mathematical machinery already developed for aiming cannons. People have been aiming cannons and plotting cannonball trajectories since the sixteenth century. We can take advantage of that existing mathematics to say exactly how, if we fired an ideal cannonball in a certain direction, it would plow into the ground. If we tried talking about rockets with realistically varying acceleration, we can’t even manage to prove that a rocket like that won’t travel around the Earth in a perfect square, because with all that realistically varying acceleration and realistic air friction it’s impossible to make any sort of definite statement one way or another. Our present understanding isn’t up to it. ALFONSO:  Okay, another question in the same vein. Why is MIRI sponsoring work on adding up lots of tiny vectors? I don’t even see what that has to do with rockets in the first place; it seems like this weird side problem in abstract math. BETH:  It’s more like… at several points in our investigation so far, we’ve run into the problem of going from a function about time-varying accelerations to a function about time-varying positions. We kept running into this problem as a blocking point in our math, in several places, so we branched off and started trying to analyze it explicitly. Since it’s about the pure mathematics of points that don’t move in discrete intervals, we call it the “logical undiscreteness” problem. Some of the ways of investigating this problem involve trying to add up lots of tiny, varying vectors to get a big vector. Then we talk about how that sum seems to change more and more slowly, approaching a limit, as the vectors get tinier and tinier and we add up more and more of them… or at least that’s one avenue of approach. ALFONSO:  I just find it hard to imagine people in future spaceplane rockets staring out their viewports and going, “Oh, no, we don’t have tiny enough vectors with which to correct our course! If only there was some way of adding up even more vectors that are even smaller! ” I’d expect future calculating machines to do a pretty good job of that already. BETH:  Again, you’re trying to draw much too straight a line between the work we’re doing now, and the implications for future rocket designs.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "It’s not like we think a rocket design will almost work, but the pilot won’t be able to add up lots of tiny vectors fast enough, so we just need a faster algorithm and then the rocket will get to the Moon. This is foundational mathematical work that we think might play a role in multiple basic concepts for understanding celestial trajectories. When we try to plot out a trajectory that goes all the way to a soft landing on a moving Moon, we feel confused and blocked. We think part of the confusion comes from not being able to go from acceleration functions to position functions, so we’re trying to resolve our confusion. ALFONSO:  This sounds suspiciously like a philosophy-of-mathematics problem, and I don’t think that it’s possible to progress on spaceplane design by doing philosophical research. The field of philosophy is a stagnant quagmire. Some philosophers still believe that going to the moon is impossible; they say that the celestial plane is fundamentally separate from the earthly plane and therefore inaccessible, which is clearly silly. Spaceplane design is an engineering problem, and progress will be made by engineers. BETH:  I agree that rocket design will be carried out by engineers rather than philosophers. I also share some of your frustration with philosophy in general. For that reason, we stick to well-defined mathematical questions that are likely to have actual answers, such as questions about how to fire a cannonball on a perfectly spherical planet with no atmosphere such that it winds up in a stable orbit. This often requires developing new mathematical frameworks. For example, in the case of the logical undiscreteness problem, we’re developing methods for translating between time-varying accelerations and time-varying positions. You can call the development of new mathematical frameworks “philosophical” if you’d like — but if you do, remember that it’s a very different kind of philosophy than the “speculate about the heavenly and earthly planes” sort, and that we’re always pushing to develop new mathematical frameworks or tools. ALFONSO:  So from the perspective of the public good, what’s a good thing that might happen if you solved this logical undiscreteness problem? BETH:  Mainly, we’d be less confused and our research wouldn’t be blocked and humanity could actually land on the Moon someday. To try and make it more concrete – though it’s hard to do that without actually knowing the concrete solution – we might be able to talk about incrementally more realistic rocket trajectories, because our mathematics would no longer break down as soon as we stopped assuming that rockets moved in straight lines.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "Our math would be able to talk about exact curves, instead of a series of straight lines that approximate the curve. ALFONSO:  An exact curve that a rocket follows? This gets me into the main problem I have with your project in general. I just don’t believe that any future rocket design will be the sort of thing that can be analyzed with absolute, perfect precision so that you can get the rocket to the Moon based on an absolutely plotted trajectory with no need to steer. That seems to me like a bunch of mathematicians who have no clue how things work in the real world, wanting everything to be perfectly calculated. Look at the way Venus moves in the sky; usually it travels in one direction, but sometimes it goes retrograde in the other direction. We’ll just have to steer as we go. BETH:  That’s not what I meant by talking about exact curves… Look, even if we can invent logical undiscreteness, I agree that it’s futile to try to predict, in advance, the precise trajectories of all of the winds that will strike a rocket on its way off the ground. Though I’ll mention parenthetically that things might actually become calmer and easier to predict, once a rocket gets sufficiently high up – ALFONSO:  Why? BETH:  Let’s just leave that aside for now, since we both agree that rocket positions are hard to predict exactly during the atmospheric part of the trajectory, due to winds and such. And yes, if you can’t exactly predict the initial trajectory, you can’t exactly predict the later trajectory. So, indeed, the proposal is definitely not to have a rocket design so perfect that you can fire it at exactly the right angle and then walk away without the pilot doing any further steering. The point of doing rocket math isn’t that you want to predict the rocket’s exact position at every microsecond, in advance. ALFONSO:  Then why obsess over pure math that’s too simple to describe the rich, complicated real universe where sometimes it rains? BETH:  It’s true that a real rocket isn’t a simple equation on a board. It’s true that there are all sorts of aspects of a real rocket’s shape and internal plumbing that aren’t going to have a mathematically compact characterization. What MIRI is doing isn’t the right degree of mathematization for all rocket engineers for all time; it’s the mathematics for us to be using right now (or so we hope).\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "To build up the field’s understanding incrementally, we need to talk about ideas whose consequences can be pinpointed precisely enough that people can analyze scenarios in a shared framework. We need enough precision that someone can say, “I think in scenario X, design Y does Z”, and someone else can say, “No, in scenario X, Y actually does W”, and the first person responds, “Darn, you’re right. Well, is there some way to change Y so that it would do Z? ” If you try to make things realistically complicated at this stage of research, all you’re left with is verbal fantasies. When we try to talk to someone with an enormous flowchart of all the gears and steering rudders they think should go into a rocket design, and we try to explain why a rocket pointed at the Moon doesn’t necessarily end up at the Moon, they just reply, “Oh, my rocket won’t do that”. Their ideas have enough vagueness and flex and underspecification that they’ve achieved the safety of nobody being able to prove to them that they’re wrong. It’s impossible to incrementally build up a body of collective knowledge that way. The goal is to start building up a library of tools and ideas we can use to discuss trajectories formally. Some of the key tools for formalizing and analyzing intuitively plausible-seeming trajectories haven’t yet been expressed using math, and we can live with that for now. We still try to find ways to represent the key ideas in mathematically crisp ways whenever we can. That’s not because math is so neat or so prestigious; it’s part of an ongoing project to have arguments about rocketry that go beyond “Does not! ” vs. “Does so! ” ALFONSO:  I still get the impression that you’re reaching for the warm, comforting blanket of mathematical reassurance in a realm where mathematical reassurance doesn’t apply. We can’t obtain a mathematical certainty of our spaceplanes being absolutely sure to reach the Moon with nothing going wrong. That being the case, there’s no point in trying to pretend that we can use mathematics to get absolute guarantees about spaceplanes. BETH:  Trust me, I am not going to feel “reassured” about rocketry no matter what math MIRI comes up with. But, yes, of course you can’t obtain a mathematical assurance of any physical proposition, nor assign probability 1 to any empirical statement. ALFONSO:  Yet you talk about proving theorems – proving that a cannonball will go in circles around the earth indefinitely, for example.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "BETH:  Proving a theorem about a rocket’s trajectory won’t ever let us feel comfortingly certain about where the rocket is actually going to end up. But if you can prove a theorem which says that your rocket would go to the Moon if it launched in a perfect vacuum, maybe you can attach some steering jets to the rocket and then have it actually go to the Moon in real life. Not with 100% probability, but with probability greater than zero. The point of our work isn’t to take current ideas about rocket aiming from a 99% probability of success to a 100% chance of success. It’s to get past an approximately 0% chance of success, which is where we are now. ALFONSO:  Zero percent? BETH:  Modulo Cromwell’s Rule, yes, zero percent. If you point a rocket’s nose at the Moon and launch it, it does not go to the Moon. ALFONSO:  I don’t think future spaceplane engineers will actually be that silly, if direct Moon-aiming isn’t a method that works. They’ll lead the Moon’s current motion in the sky, and aim at the part of the sky where Moon will appear on the day the spaceplane is a Moon’s distance away. I’m a bit worried that you’ve been talking about this problem so long without considering such an obvious idea. BETH:  We considered that idea very early on, and we’re pretty sure that it still doesn’t get us to the Moon. ALFONSO:  What if I add steering fins so that the rocket moves in a more curved trajectory? Can you prove that no version of that class of rocket designs will go to the Moon, no matter what I try? BETH:  Can you sketch out the trajectory that you think your rocket will follow? ALFONSO:  It goes from the Earth to the Moon. BETH:  In a bit more detail, maybe? ALFONSO:  No, because in the real world there are always variable wind speeds, we don’t have infinite fuel, and our spaceplanes don’t move in perfectly straight lines. BETH:  Can you sketch out a trajectory that you think a simplified version of your rocket will follow, so we can examine the assumptions your idea requires? ALFONSO:  I just don’t believe in the general methodology you’re proposing for spaceplane designs. We’ll put on some steering fins, turn the wheel as we go, and keep the Moon in our viewports.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "If we’re off course, we’ll steer back. BETH:  … We’re actually a bit concerned that standard steering fins may stop working once the rocket gets high enough, so you won’t actually find yourself able to correct course by much once you’re in the celestial reaches – like, if you’re already on a good course, you can correct it, but if you screwed up, you won’t just be able to turn around like you could turn around an airplane – ALFONSO:  Why not? BETH:  We can go into that topic too; but even given a simplified model of a rocket that you could steer, a walkthrough of the steps along the path that simplified rocket would take to the Moon would be an important step in moving this discussion forward. Celestial rocketry is a domain that we expect to be unusually difficult – even compared to building rockets on Earth, which is already a famously hard problem because they usually just explode. It’s not that everything has to be neat and mathematical. But the overall difficulty is such that, in a proposal like “lead the moon in the sky,” if the core ideas don’t have a certain amount of solidity about them, it would be equivalent to firing your rocket randomly into the void. If it feels like you don’t know for sure whether your idea works, but that it might work; if your idea has many plausible-sounding elements, and to you it feels like nobody has been able to convincingly explain to you how it would fail; then, in real life, that proposal has a roughly 0% chance of steering a rocket to the Moon. If it seems like an idea is extremely solid and clearly well-understood, if it feels like this proposal should definitely take a rocket to the Moon without fail in good conditions, then maybe under the best-case conditions we should assign an 85% subjective credence in success, or something in that vicinity. ALFONSO:  So uncertainty automatically means failure? This is starting to sound a bit paranoid, honestly. BETH:  The idea I’m trying to communicate is something along the lines of, “If you can reason rigorously about why a rocket should definitely work in principle, it might work in real life, but if you have anything less than that, then it definitely won’t work in real life”. I’m not asking you to give me an absolute mathematical proof of empirical success.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - "ONSO:  Why not? BETH:  We can go into that topic too; but even given a simplified model of a rocket that you could steer, a walkthrough of the steps along the path that simplified rocket would take to the Moon would be an important step in moving this discussion forward. Celestial rocketry is a domain that we expect to be unusually difficult – even compared to building rockets on Earth, which is already a famously hard problem because they usually just explode. It’s not that everything has to be neat and mathematical. But the overall difficulty is such that, in a proposal like “lead the moon in the sky,” if the core ideas don’t have a certain amount of solidity about them, it would be equivalent to firing your rocket randomly into the void. If it feels like you don’t know for sure whether your idea works, but that it might work; if your idea has many plausible-sounding elements, and to you it feels like nobody has been able to convincingly explain to you how it would fail; then, in real life, that proposal has a roughly 0% chance of steering a rocket to the Moon. If it seems like an idea is extremely solid and clearly well-understood, if it feels like this proposal should definitely take a rocket to the Moon without fail in good conditions, then maybe under the best-case conditions we should assign an 85% subjective credence in success, or something in that vicinity. ALFONSO:  So uncertainty automatically means failure? This is starting to sound a bit paranoid, honestly. BETH:  The idea I’m trying to communicate is something along the lines of, “If you can reason rigorously about why a rocket should definitely work in principle, it might work in real life, but if you have anything less than that, then it definitely won’t work in real life”. I’m not asking you to give me an absolute mathematical proof of empirical success.I’m asking you to give me something more like a sketch for how a simplified version of your rocket could move, that’s sufficiently determined in its meaning that you can’t just come back and say “Oh, I didn’t mean that” every time someone tries to figure out what it actually does or pinpoint a failure mode. This isn’t an unreasonable demand that I’m imposing to make it impossible for any ideas to pass my filters. It’s the primary bar all of us have to pass to contribute to collective progress in this field. And a rocket design which can’t even pass that conceptual bar has roughly a 0% chance of landing softly on the Moon.\n", - "\n", - " The post The Rocket Alignment Problem appeared first on Machine Intelligence Research Institute.\n", - " - Title: The Rocket Alignment Problem, Author: Eliezer Yudkowsky, Date published: Wed, 03 Oct 2018, URL: https://intelligence.org/2018/10/03/rocket-alignment/\n", - " Jonathan Millen on covert channel communication\n", - "\n", - " Jonathan Millen started work at the MITRE Corporation in 1969, after graduation from Rensselaer Polytechnic Institute with a Ph.D. in Mathematics. He retired from MITRE in 2012 as a Senior Principal in the Information Security Division. From 1997 to 2004 he enjoyed an interlude as a Senior Computer Scientist in the SRI International Computer Science Laboratory. He has given short courses at RPI Hartford, University of Bologna Summer School, ETH Zurich, and Taiwan University of Science and Technology. He organized the IEEE Computer Security Foundations Symposium (initially a workshop) in 1988, and co-founded (with S. Jajodia) the Journal of Computer Security in 1992. He has held positions as General and Program Chair of the IEEE Security and Privacy Symposium, Chair of the IEEE Computer Society Technical Committee on Security and Privacy, and associate editor of the ACM Transactions on Information and System Security. The theme of his computer security interests is verification of formal specifications, of security kernels and cryptographic protocols. At MITRE he supported the DoD Trusted Product Evaluation Program, and later worked on the application of Trusted Platform Modules. He wrote several papers on information flow as applied to covert channel detection and measurement. His 2001 paper (with V. Shmatikov) on the Constraint Solver for protocol analysis received the SIGSAC Test of Time award in 2011. He received the ACM SIGSAC Outstanding Innovation award in 2009.\n", - "\n", - " Luke Muehlhauser: Since you were a relatively early researcher in the field of covert channel communication, I’d like to ask you about the field’s early days, which are usually said to have begun with Lampson (1973). Do you know when the first covert channel attack was uncovered “in the wild”? My impression is that Lampson identified the general problem a couple decades before it was noticed being exploited in the wild; is that right?\n", - "\n", - " Jonathan Millen: We might never know when real covert channel attacks were first noticed, or when they first occurred. When information is stolen by covert channel, the original data is still in place, so the theft can go unnoticed. Even if an attack is discovered, the victims are as reluctant as the perpetrators to acknowledge it. This is certainly the case with classified information, since a known attack is often classified higher than the information it compromises. The only evidence I have of real attacks before 1999 is from Robert Morris (senior), a pioneer in UNIX security, and for a while the Chief Scientist of the National Computer Security Center, which was organizationally within NSA.\n", - " - Title: Jonathan Millen on covert channel communication, Author: Luke Muehlhauser, Date published: Sat, 12 Apr 2014, URL: https://intelligence.org/2014/04/12/jonathan-millen/\n", - "He stated at a security workshop that there had been real attacks. He wouldn’t say anything more; it was probably difficult enough to get clearance for that much.\n", - "\n", - " Luke: From your own perspective, what are some of the most interesting developments in covert channel communication since you wrote “20 Years of Covert Channel Modeling and Analysis” in 1999?\n", - "\n", - " Jonathan: That’s a tough question, for several reasons. Like many research areas that have been in existence for many years, covert channel research has splintered into several specialties. Some of the subtopics are noninterference models, language-based information control via type theory, steganography, and so on. While specialization is necessary for deep progress, I wish there were more attention given to the pragmatic “so what” questions about covert channels. For example, information flow security is often defined in terms of noninterference, which can, unfortunately, be defined in several subtly different ways. All of them rely on some form of behavioral equivalence, a strong condition guaranteeing that an unauthorized observer cannot tell when a more privileged user is present. Even one bit, or a fraction of a bit, of information about the protected party’s data or actions is considered too much. Yet, what we really want to know is whether a recurrent channel can be set up, yielding a large or unbounded amount of information. Furthermore, information in the Shannon sense may or may not be useful. One paper that attracted my attention is “Quantifying information flow with beliefs”, by Clarkson, Myers, and Schneider, 2009, because it dealt with the accuracy of information. Another pragmatic issue is the trustworthiness of the system functionality. In general, I am distrustful of security approaches that depend on access control software or the design of the language in which it is written. Even the firmware and hardware below the operating system kernel is suspect. In the last few years, my main research interest was in trusted computing, as supported by trusted platform modules, with their capability — available but largely unused in modern platforms — to cryptographically check the integrity of system software by a bootstrap sequence starting with the tamper-resistant TPM. And still, having done that, we know only that the system software is genuine, not that it is correct or free from covert channels. It is encouraging that work in software and hardware verification continues, but commonly used systems are still beyond its reach. Complexity is the enemy of security. The more you have to depend on, the less likely it is that you can understand it well enough to exclude vulnerabilities.\n", - " - Title: Jonathan Millen on covert channel communication, Author: Luke Muehlhauser, Date published: Sat, 12 Apr 2014, URL: https://intelligence.org/2014/04/12/jonathan-millen/\n", - "A couple of corollaries: the more complex your security policy is, the less robust it is, because more complex software is needed for it. And the more specialized a system component is to support security, the more attractive it is as a target for an attack. I suppose that the optimistic view of this is that if we use a typically complex modern system, covert channels are the least of our worries!\n", - "\n", - " Luke: From your many decades of experience in computer security, do you know of cases where someone was worried about a computer security or safety challenge that wasn’t imminent but maybe one or two decades away, and they decided to start doing research to prepare for that challenge anyway — e.g. perhaps because they expected the solution would require a decade or two of “serial” research and/or engineering work, with each piece building on the ones before it, and they wanted to be prepared to meet the challenge near when it arrived? Lampson’s early identification of the “confinement problem” looks to me like it might be one such example, but maybe I’m misreading the history there.\n", - "\n", - " Jonathan: In computer security, what usually happens is that someone realizes that a vulnerability already exists, but it is not clear how long it will take for a malicious party to take advantage of it. Another factor delaying the onset of relevant research and development is that long-term efforts aimed at removing vulnerabilities are risky and their results may be inadequate or eclipsed by different problems in the future. If the difficulty is social, such as getting standards updated, the difficulty is real, but the basic engineering knowledge is mostly available early on. For example, the TCP sequence number attack for session hijacking was pointed out by Morris in 1985, taken advantage of by Mitnick in 1995, led to Bellovin’s standards-related recommendations in 1996, and only recently has the availability of encryption-based authentication and IPv6 brought a widespread solution within reach. If the difficulty is in the engineering, such as the development of security kernels and mandatory access policies to defeat Trojan horses, or the installation and use of TPMs, there is an industry to be moved, one driven by the need for commercial viability of new products. And after all that effort, some vulnerabilities remain. Computer scientists can help by developing powerful general-purpose tools and techniques with fundamental computer science goals — tools that are not problem-dependent. By this, I mean such things as verification tools and model checkers, as well as encryption algorithms and applications. Such tools can ease the engineering burden when new problems and design ideas emerge.\n", - " - Title: Jonathan Millen on covert channel communication, Author: Luke Muehlhauser, Date published: Sat, 12 Apr 2014, URL: https://intelligence.org/2014/04/12/jonathan-millen/\n", - "Development of these tools deserve decades of research building on prior work, and those who are capable of conducting this activity successfully should be encouraged and supported.\n", - "\n", - " Luke: You mention verification tools and model checkers. I was recently speaking to a significant figure in the computer security and safety community (I don’t have permission to give her name) about the common claim that formal methods can catch corner bugs which are unlikely to be caught by testing alone. She mentioned that if the formal methods community has ever subjected formal methods to careful experimentation to demonstrate that verification really does catch bugs that aren’t likely to be found by testing, she wasn’t aware of it. Are you aware of any such demonstrations? If not, how would you describe the value in formal methods for safety and security purposes?\n", - "\n", - " Jonathan: It seems unfair to me to ask for careful experimentation to justify formal methods in comparison to testing. One reason for this that was pointed out when formal methods were being recommended for security kernel verification, is that very few security kernels are actually developed, so any experimental results are not going to be statistically significant. One experiment that was done in the very beginning of the history of formal specifications was by David Parnas, credited by some with inventing the notion of “formal nonprocedural specification”. 1 In fact, many of us referred to such specifications at first as “Parnas specifications”, but he let it be known (through Peter Neumann) that he disapproved of that. Parnas’ experiment was a small classroom experiment designed to investigate, not the efficacy of verification of formal specifications, but rather the plausible idea that specifications determine implementations. Parnas’ classroom experiment, in which several groups were given the same specification to implement, disproved that (or at least cast doubt on that) because different groups implemented their specified task in very different ways, despite the very detailed constraints on input-output behavior in the specification. (I heard the experiment described at a conference presentation by Parnas, but I can’t find the paper or the citation now. ) This result turned out to be useful to me a few years later when DoD sponsors argued that analysis of the formal specifications of security kernels should be classified higher than the programs themselves, even when the analysis proved the absence of access control violations and storage channels, because the specifications allowed readers to deduce features of the implementation that would tell them about timing and other channels. I pointed out that Parnas had given evidence that their fears were not justified.\n", - " - Title: Jonathan Millen on covert channel communication, Author: Luke Muehlhauser, Date published: Sat, 12 Apr 2014, URL: https://intelligence.org/2014/04/12/jonathan-millen/\n", - "One indicator of the value of formal methods, model checking in particular, is the fact that it has been used routinely in the design of CPUs, which have rather complicated internal strategies for managing memory and register caching. (One citation I found with a quick Google to confirm this is “Fifteen years of formal property verification in Intel”, by Limor Fix, of Intel Research Pittsburgh, published in a 2008 Springer-Verlag book, 25 Years of Model Checking.\n", - "\n", - " Luke: You also wrote that “Computer scientists can help by developing powerful general-purpose tools and techniques… [e.g.] verification tools and model checkers, as well as encryption algorithms and applications… Development of these tools deserve decades of research building on prior work, and those who are capable of conducting this activity successfully should be encouraged and supported”. What are some specific examples of tools for software system safety or security that you wish were receiving more development talent and funding than is currently the case? E.g. probabilistic model-checkers, or verified libraries for program synthesis, or whatever you think is most urgent or underfunded.\n", - "\n", - " Jonathan: My gut reaction is, all of the above. When it comes to specific tools, each of us has preferences based on what we needed, what we have used, and what is most accessible. When I was at SRI International, I used the PVS theorem prover, and later, SRI’s SAL environment, which hosts several flavors of model checking algorithms. Both of those have benefited from the highest levels of talent; the challenge is that the job is never finished, especially if it is a thriving, well-used system. Funding sources typically emphasize urgent applications or brand new ideas, as they should, but they tend to neglect the steady, unromantic work needed to keep a software suite up to date with respect to the evolution of hardware, operating systems, algorithmic advances, and so on. And tools that support formal methods do not have the same opportunities for income as Windows, Apple’s OS X, and other commercial systems, even with a strong user community. Incidentally, although I have emphasized general-purpose tools, that does not mean that I am against tools designed for particular applications such as security. After all, I spent a fair amount of time on a protocol security analyzer (called the Constraint Solver). Specialized tools, however, benefit in brevity and readability from being built on top of an expressive language with powerful primitives — for my application, Prolog, specifically SWI-Prolog.\n", - " - Title: Jonathan Millen on covert channel communication, Author: Luke Muehlhauser, Date published: Sat, 12 Apr 2014, URL: https://intelligence.org/2014/04/12/jonathan-millen/\n", - " wrote that “Computer scientists can help by developing powerful general-purpose tools and techniques… [e.g.] verification tools and model checkers, as well as encryption algorithms and applications… Development of these tools deserve decades of research building on prior work, and those who are capable of conducting this activity successfully should be encouraged and supported”. What are some specific examples of tools for software system safety or security that you wish were receiving more development talent and funding than is currently the case? E.g. probabilistic model-checkers, or verified libraries for program synthesis, or whatever you think is most urgent or underfunded.\n", - "\n", - " Jonathan: My gut reaction is, all of the above. When it comes to specific tools, each of us has preferences based on what we needed, what we have used, and what is most accessible. When I was at SRI International, I used the PVS theorem prover, and later, SRI’s SAL environment, which hosts several flavors of model checking algorithms. Both of those have benefited from the highest levels of talent; the challenge is that the job is never finished, especially if it is a thriving, well-used system. Funding sources typically emphasize urgent applications or brand new ideas, as they should, but they tend to neglect the steady, unromantic work needed to keep a software suite up to date with respect to the evolution of hardware, operating systems, algorithmic advances, and so on. And tools that support formal methods do not have the same opportunities for income as Windows, Apple’s OS X, and other commercial systems, even with a strong user community. Incidentally, although I have emphasized general-purpose tools, that does not mean that I am against tools designed for particular applications such as security. After all, I spent a fair amount of time on a protocol security analyzer (called the Constraint Solver). Specialized tools, however, benefit in brevity and readability from being built on top of an expressive language with powerful primitives — for my application, Prolog, specifically SWI-Prolog.A word of caution, though: it’s one thing to build an analysis tool with a complex system under it, and another to build a supposedly secure system that way. A lesson learned from studying covert channels is that the whole system under the secure interface is a potential source of vulnerabilities. So how can we build secure systems? The main requirement, I think, is simple: don’t let the enemy program your computer (the computer holding your data). But how can we prevent that? That’s the hard part.\n", - "\n", - " Luke: Thanks, Jonathan! Parnas (1972). The post Jonathan Millen on covert channel communication appeared first on Machine Intelligence Research Institute.\n", - " - Title: Jonathan Millen on covert channel communication, Author: Luke Muehlhauser, Date published: Sat, 12 Apr 2014, URL: https://intelligence.org/2014/04/12/jonathan-millen/\n", - " Probabilistic Metamathematics and the Definability of Truth\n", - "\n", - " On October 15th, Paul Christiano presented “Probabilistic metamathematics and the definability of truth” at Harvard University as part of Logic at Harvard (details here). As explained here, Christiano came up with the idea for this approach, and it was developed further at a series of MIRI research workshops.\n", - "\n", - " The video is occasionally blurry due to camera problems, but is still clear enough to watch. The post Probabilistic Metamathematics and the Definability of Truth appeared first on Machine Intelligence Research Institute.\n", - " - Title: Probabilistic Metamathematics and the Definability of Truth, Author: Luke Muehlhauser, Date published: Wed, 23 Oct 2013, URL: https://intelligence.org/2013/10/23/probabilistic-metamathematics-and-the-definability-of-truth/\n" - ] - } - ], + "outputs": [], "source": [ - "for data in dataset.embed_split:\n", - " print(data)" + "#for data in dataset.embedding_strings:\n", + "# print(data)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Title: What is ambitious value learning?\n", - "Author: Rohin Shah\n", - "Date published: 2018-11-01\n", - "URL: https://www.lesswrong.com/posts/5eX8ko7GCxwR5N9mN/what-is-ambitious-value-learning\n", - "Tags: Value Learning/AI\n", - "Text: I think of ambitious value learning as a proposed solution to the specification problem, which I define as the problem of *defining* the behavior that we would want to see from our AI system. I italicize \"defining\" to emphasize that this is *not* the problem of actually *computing* behavior that we want to see—that’s the full AI safety problem. Here we are allowed to use hopelessly impractical schemes, as long as the resulting definition would allow us to *in theory* compute the behavior that an AI system would take, perhaps with assumptions like infinite computing power or arbitrarily many queries to a human. (Although we do prefer specifications that seem like they could admit an efficient implementation.) In terms of DeepMind’s classification, we are looking for a design specification that exactly matches the ideal specification. HCH and indirect normativity are examples of attempts at such specifications.\n", - "We will consider a model in which our AI system is maximizing the expected utility of some *explicitly* represented utility function that can depend on history. (It does not matter materially whether we consider utility functions or reward functions, as long as they can depend on history.) The utility function may be learned from data, or designed by hand, but it must be an explicit part of the AI that is then maximized.\n", - "I will not justify this model for now, but simply assume it by fiat and see where it takes us. I’ll note briefly that this model is often justified by the VNM utility theorem and AIXI, and as the natural idealization of reinforcement learning, which aims to maximize the expected sum of rewards, although typically rewards in RL depend only on states.\n", - "A lot of conceptual arguments, as well as experiences with specification gaming, suggest that we are unlikely to be able to simply think hard and write down a good specification, since even small errors in specifications can lead to bad results. However, machine learning is particularly good at narrowing down on the correct hypothesis among a vast space of possibilities using data, so perhaps we could determine a good specification from some suitably chosen source of data? This leads to the idea of ambitious value learning, where we *learn* an explicit utility function from human behavior for the AI to maximize.\n", - "This is very related to inverse reinforcement learning (IRL) in the machine learning literature, though not all work on IRL is relevant to ambitious value learning. For example, much work on IRL is aimed at *imitation learning*, which would in the best case allow you to match human performance, but not to exceed it. Ambitious value learning is, well, more ambitious—it aims to learn a utility function that captures \"what humans care about\", so that an AI system that optimizes this utility function more capably can *exceed* human performance, making the world better for humans than they could have done themselves.\n", - "It may sound like we would have solved the entire AI safety problem if we could do ambitious value learning—surely if we have a good utility function we would be done. Why then do I think of it as a solution to just the specification problem? This is because ambitious value learning by itself would not be enough for safety, except under the assumption of as much compute and data as desired. These are really powerful assumptions—for example, I’m assuming you can get data where you put a human in an arbitrarily complicated simulated environment with fake memories of their life so far and see what they do. This allows us to ignore many things that would likely be a problem in practice, such as:\n", - "\n", - " - Attempting to use the utility function to choose actions before it has converged\n", - "\n", - " - Distributional shift causing the learned utility function to become invalid\n", - "\n", - " - Local minima preventing us from learning a good utility function, or from optimizing the learned utility function correctly\n", - "The next few posts in this sequence will consider the suitability of ambitious value learning as a solution to the specification problem. Most of them will consider whether ambitious value learning is possible in the setting above (infinite compute and data). One post will consider practical issues with the application of IRL to infer a utility function suitable for ambitious value learning, while still assuming that the resulting utility function can be perfectly maximized (which is equivalent to assuming infinite compute and a perfect model of the environment *after* IRL has run).\n", - "\n" + "Title: The innocent self\n", + "Author: Richard_Ngo\n", + "Date published: 2022-02-13\n", + "URL: https://www.lesswrong.com/posts/fWTkd2RSx3jcvfLCt/the-innocent-self\n", + "Tags: World Modeling\n" ] } ], "source": [ "article_num = 0\n", - "print(f\"Title: {dataset.data[article_num][0]}\")\n", - "print(f\"Author: {dataset.data[article_num][1]}\")\n", - "print(f\"Date published: {dataset.data[article_num][2]}\")\n", - "print(f\"URL: {dataset.data[article_num][3]}\")\n", - "print(f\"Tags: {dataset.data[article_num][4]}\")\n", - "print(f\"Text: {dataset.data[article_num][5]}\")" + "\n", + "example_article_metadata = dataset.metadata[article_num]\n", + "print(f\"Title: {example_article_metadata[0]}\")\n", + "print(f\"Author: {example_article_metadata[1]}\")\n", + "print(f\"Date published: {example_article_metadata[2]}\")\n", + "print(f\"URL: {example_article_metadata[3]}\")\n", + "print(f\"Tags: {example_article_metadata[4]}\")\n", + "#print(f\"Text: {example_article_metadata[5]}\")\n" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Articles count: 8\n", - "Num of each source: {'alignment forum': 4, 'manual': 0, 'waitbutwhy.com': 0, 'https://aiimpacts.org': 0, 'arbital.com': 0, 'https://intelligence.org': 4, 'reports': 0, 'https://aisafety.camp': 0, 'curriculum': 0, 'https://www.yudkowsky.net': 0, 'distill': 0, 'total': 8}\n", - "Num chars: 72919\n", - "Num words: 11997\n", - "Num sentences: 523\n", - "Num blocks: 34\n" + "Articles count: 12\n", + "Num of each source: defaultdict(, {'alignment forum': 1, 'lesswrong': 11, 'manual': 0, 'waitbutwhy.com': 0, 'https://aiimpacts.org': 0, 'arbital.com': 0, 'https://intelligence.org': 0, 'reports': 0, 'https://aisafety.camp': 0, 'curriculum': 0, 'https://www.yudkowsky.net': 0, 'distill': 0})\n", + "Num articles: 12\n", + "Num chars: 50028\n", + "Num words: 8119\n", + "Num sentences: 411\n", + "Num blocks: 28\n" ] } ], "source": [ - "print(f\"Articles count: {len(dataset.data)}\")\n", - "print(f\"Num of each source: {dataset.num_articles}\")\n", + "print(f\"Articles count: {len(dataset.metadata)}\")\n", + "print(f\"Num of each source: {dataset.articles_count}\")\n", "print(f\"Num chars: {dataset.total_char_count}\")\n", "print(f\"Num words: {dataset.total_word_count}\")\n", "print(f\"Num sentences: {dataset.total_sentence_count}\")\n", @@ -1440,7 +1095,7 @@ } ], "source": [ - "for embed in dataset.embed_split:\n", + "for embed in dataset.embedding_strings:\n", " print(embed)\n", " print()" ] @@ -1471,7 +1126,7 @@ " def __init__(self,\n", " dataset: Dataset, # Dataset object containing the data.\n", " ):\n", - " self.dataset = dataset\n", + " self.metadataset = dataset\n", " \n", " # @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(10))\n", " def get_embedding(self, text: str) -> np.ndarray:\n", @@ -1487,9 +1142,9 @@ " # Receives a query (str) and returns the top k blocks that are most semantically similar to the query.\n", " # Each tuple contains the title of an article, its URL, and text.\n", " query_embedding = self.get_embedding(query)\n", - " similarities = np.dot(self.dataset.embeddings, query_embedding)\n", + " similarities = np.dot(self.metadataset.embeddings, query_embedding)\n", " top_k_indices = np.argsort(similarities)[::-1][:k]\n", - " top_k = [self.dataset.embed_split[i] for i in top_k_indices]\n", + " top_k = [self.metadataset.embedding_strings[i] for i in top_k_indices]\n", " return top_k\n", " \n", " def construct_messages(self, question: str, blocks: List[str] = None, mode: str = \"balanced\") -> str:\n",