added marple paper

content/publication/franken2024sami.md

@@ -9,9 +9,9 @@
 title = "Self-supervised alignment with mutual information: Learning to follow principles without preference labels"
 date = "2024-04-22"
 authors = ["J. Fränken","E. Zelikman","R. Rafailov","K. Gandhi","T. Gerstenberg","N. D. Goodman"]
-publication_types = ["1"]
-publication_short = "_arXiv_"
-publication = "Fränken, J., Zelikman, E., Rafailov, R., Gandhi, K., Gerstenberg, T., Goodman, N. D. (2024). Self-supervised alignment with mutual information: Learning to follow principles without preference labels. _arXiv_."
+publication_types = ["3"]
+publication_short = "_Advances in Neural Information Processing Systems_"
+publication = "Fränken, J., Zelikman, E., Rafailov, R., Gandhi, K., Gerstenberg, T., Goodman, N. D. (2024). Self-supervised alignment with mutual information: Learning to follow principles without preference labels. _Advances in Neural Information Processing Systems_."
 abstract = "When prompting a language model (LM), users frequently expect the model to adhere to a set of behavioral principles across diverse tasks, such as producing insightful content while avoiding harmful or biased language. Instilling such principles into a model can be resource-intensive and technically challenging, generally requiring human preference labels or examples. We introduce SAMI, a method for teaching a pretrained LM to follow behavioral principles that does not require any preference labels or demonstrations. SAMI is an iterative algorithm that finetunes a pretrained LM to increase the conditional mutual information between constitutions and self-generated responses given queries from a datasest. On single-turn dialogue and summarization, a SAMI-trained mistral-7b outperforms the initial pretrained model, with win rates between 66% and 77%. Strikingly, it also surpasses an instruction-finetuned baseline (mistral-7b-instruct) with win rates between 55% and 57% on single-turn dialogue. SAMI requires a 'principle writer' model; to avoid dependence on stronger models, we further evaluate aligning a strong pretrained model (mixtral-8x7b) using constitutions written by a weak instruction-finetuned model (mistral-7b-instruct). The SAMI-trained mixtral-8x7b outperforms both the initial model and the instruction-finetuned model, achieving a 65% win rate on summarization. Our results indicate that a pretrained LM can learn to follow constitutions without using preference labels, demonstrations, or human oversight."
 image_preview = ""
 selected = false

content/publication/gandhi2024affective.md

@@ -16,7 +16,7 @@ abstract = "Understanding emotions is fundamental to human interaction and exper
 image_preview = ""
 selected = false
 projects = []
-#url_pdf = "papers/gandhi2024affective.pdf"
+url_pdf = "papers/gandhi2024affective.pdf"
 url_preprint = "https://arxiv.org/abs/2409.11733"
 url_code = ""
 url_dataset = ""

content/publication/jin2024marple.md

@@ -0,0 +1,33 @@
++++
+# 0 -> 'Forthcoming',
+# 1 -> 'Preprint',
+# 2 -> 'Journal',
+# 3 -> 'Conference Proceedings',
+# 4 -> 'Book chapter',
+# 5 -> 'Thesis'
+
+title = "MARPLE: A Benchmark for Long-Horizon Inference"
+date = "2024-10-04"
+authors = ["E. Jin","Z. Huang","J. Fränken","W. Liu","H. Cha","E. Brockbank","S. Wu","R. Zhang","J. Wu","T. Gerstenberg"]
+publication_types = ["3"]
+publication_short = "_Advances in Neural Information Processing Systems_"
+publication = "Jin, E., Huang, Z., Fränken, J., Liu, W., Cha, H., Brockbank, E., Wu, S., Zhang, R., Wu, J., Gerstenberg, T. (2024). MARPLE: A Benchmark for Long-Horizon Inference. _Advances in Neural Information Processing Systems_."
+abstract = "Reconstructing past events requires reasoning across long time horizons. To figure out what happened, we need to use our prior knowledge about the world and human behavior and draw inferences from various sources of evidence including visual, language, and auditory cues. We introduce MARPLE, a benchmark for evaluating long-horizon inference capabilities using multi-modal evidence. Our benchmark features agents interacting with simulated households, supporting vision, language, and auditory stimuli, as well as procedurally generated environments and agent behaviors. Inspired by classic ``whodunit'' stories, we ask AI models and human participants to infer which agent caused a change in the environment based on a step-by-step replay of what actually happened. The goal is to correctly identify the culprit as early as possible. Our findings show that human participants outperform both traditional Monte Carlo simulation methods and an LLM baseline (GPT-4) on this task. Compared to humans, traditional inference models are less robust and performant, while GPT-4 has difficulty comprehending environmental changes. We analyze what factors influence inference performance and ablate different modes of evidence, finding that all modes are valuable for performance. Overall, our experiments demonstrate that the long-horizon, multimodal inference tasks in our benchmark present a challenge to current models. Project website: https://marple-benchmark.github.io/."
+image_preview = ""
+selected = false
+projects = []
+url_pdf = "papers/jin2024marple.pdf"
+url_preprint = "http://arxiv.org/abs/2410.01926"
+url_code = ""
+url_dataset = ""
+url_slides = ""
+url_video = ""
+url_poster = ""
+url_source = ""
+url_custom = [{name = "Project website", url = "https://marple-benchmark.github.io/"}]
+math = true
+highlight = true
+[header]
+# image = "publications/jin2024marple.png"
+caption = ""
++++

docs/404.html

@@ -237,6 +237,10 @@ <h1>Page not found</h1>
 
   <h2>Publications</h2>
 
+  <ul>
+    <li><a href="https://cicl.stanford.edu/publication/jin2024marple/">MARPLE: A Benchmark for Long-Horizon Inference</a></li>
+  </ul>
+
   <ul>
     <li><a href="https://cicl.stanford.edu/publication/beller2024causation/">Causation, Meaning, and Communication</a></li>
   </ul>
@@ -253,10 +257,6 @@ <h2>Publications</h2>
     <li><a href="https://cicl.stanford.edu/publication/gerstenberg2024counterfactual/">Counterfactual simulation in causal cognition</a></li>
   </ul>
 
-  <ul>
-    <li><a href="https://cicl.stanford.edu/publication/amemiya2024disagreement/">Children use disagreement to infer what happened</a></li>
-  </ul>
-
 
 
 

docs/bibtex/cic_papers.bib

@@ -1,13 +1,24 @@
 %% This BibTeX bibliography file was created using BibDesk.
 %% https://bibdesk.sourceforge.io/
 
-%% Created for Tobias Gerstenberg at 2024-09-20 15:09:53 -0700 
+%% Created for Tobias Gerstenberg at 2024-10-04 08:51:52 -0700 
 
 
 %% Saved with string encoding Unicode (UTF-8) 
 
 
 
+@article{jin2024marple,
+	abstract = {Reconstructing past events requires reasoning across long time horizons. To figure out what happened, we need to use our prior knowledge about the world and human behavior and draw inferences from various sources of evidence including visual, language, and auditory cues. We introduce MARPLE, a benchmark for evaluating long-horizon inference capabilities using multi-modal evidence. Our benchmark features agents interacting with simulated households, supporting vision, language, and auditory stimuli, as well as procedurally generated environments and agent behaviors. Inspired by classic ``whodunit'' stories, we ask AI models and human participants to infer which agent caused a change in the environment based on a step-by-step replay of what actually happened. The goal is to correctly identify the culprit as early as possible. Our findings show that human participants outperform both traditional Monte Carlo simulation methods and an LLM baseline (GPT-4) on this task. Compared to humans, traditional inference models are less robust and performant, while GPT-4 has difficulty comprehending environmental changes. We analyze what factors influence inference performance and ablate different modes of evidence, finding that all modes are valuable for performance. Overall, our experiments demonstrate that the long-horizon, multimodal inference tasks in our benchmark present a challenge to current models. Project website: https: //marple-benchmark.github.io/.},
+	annote = {Comment: NeurIPS 2024. First two authors contributed equally. Project page: https://marple-benchmark.github.io/},
+	author = {Jin, Emily and Huang, Zhuoyi and Fr{\"a}nken, Jan-Philipp and Liu, Weiyu and Cha, Hannah and Brockbank, Erik and Wu, Sarah and Zhang, Ruohan and Wu, Jiajun and Gerstenberg, Tobias},
+	date-added = {2024-10-04 08:51:51 -0700},
+	date-modified = {2024-10-04 08:51:51 -0700},
+	journal = {arXiv},
+	note = {http://arxiv.org/abs/2410.01926},
+	title = {{MARPLE: A Benchmark for Long-Horizon Inference}},
+	year = {2024}}
+
 @article{gandhi2024affective,
 	abstract = {Understanding emotions is fundamental to human interaction and experience. Humans easily infer emotions from situations or facial expressions, situations from emotions, and do a variety of other affective cognition. How adept is modern AI at these inferences? We introduce an evaluation framework for testing affective cognition in foundation models. Starting from psychological theory, we generate 1,280 diverse scenarios exploring relationships between appraisals, emotions, expressions, and outcomes. We evaluate the abilities of foundation models (GPT-4, Claude-3, Gemini-1.5-Pro) and humans (N = 567) across carefully selected conditions. Our results show foundation models tend to agree with human intuitions, matching or exceeding interparticipant agreement. In some conditions, models are ``superhuman'' -- they better predict modal human judgements than the average human. All models benefit from chain-of-thought reasoning. This suggests foundation models have acquired a human-like understanding of emotions and their influence on beliefs and behavior.},
 	author = {Kanishk Gandhi and Zoe Lynch and Jan-Philipp Fr{\"a}nken and Kayla Patterson and Sharon Wambu and Tobias Gerstenberg and Desmond C. Ong and Noah D. Goodman},

docs/index.html

@@ -110,7 +110,7 @@
   <meta property="og:description" content="">
   <meta property="og:locale" content="en-us">
 
-  <meta property="og:updated_time" content="2024-09-20T00:00:00&#43;00:00">
+  <meta property="og:updated_time" content="2024-10-04T00:00:00&#43;00:00">
 
 
 

docs/index.xml

@@ -6,9 +6,18 @@
     <generator>Hugo -- gohugo.io</generator>
     <language>en-us</language>
     <copyright>&amp;copy; 2024 Tobias Gerstenberg</copyright>
-    <lastBuildDate>Fri, 20 Sep 2024 00:00:00 +0000</lastBuildDate>
+    <lastBuildDate>Fri, 04 Oct 2024 00:00:00 +0000</lastBuildDate>
     <atom:link href="/" rel="self" type="application/rss+xml" />
 
+    <item>
+      <title>MARPLE: A Benchmark for Long-Horizon Inference</title>
+      <link>https://cicl.stanford.edu/publication/jin2024marple/</link>
+      <pubDate>Fri, 04 Oct 2024 00:00:00 +0000</pubDate>
+
+      <guid>https://cicl.stanford.edu/publication/jin2024marple/</guid>
+      <description></description>
+    </item>
+
     <item>
       <title>Causation, Meaning, and Communication</title>
       <link>https://cicl.stanford.edu/publication/beller2024causation/</link>
@@ -135,14 +144,5 @@
       <description></description>
     </item>
 
-    <item>
-      <title>STaR-GATE: Teaching Language Models to Ask Clarifying Questions</title>
-      <link>https://cicl.stanford.edu/publication/andukuri2024stargate/</link>
-      <pubDate>Sun, 31 Mar 2024 00:00:00 +0000</pubDate>
-
-      <guid>https://cicl.stanford.edu/publication/andukuri2024stargate/</guid>
-      <description></description>
-    </item>
-
   </channel>
 </rss>

docs/member/tobias_gerstenberg/index.html

@@ -356,6 +356,53 @@ <h2 id="publications">Publications</h2>
 
 
        <div class="pub-list-item" style="margin-bottom: 1rem" itemscope itemtype="http://schema.org/CreativeWork">
+  <span itemprop="author">
+    E. Jin, Z. Huang, J. Fränken, W. Liu, H. Cha, E. Brockbank, S. Wu, R. Zhang, J. Wu, T. Gerstenberg</span>
+
+    (2024).
+
+  <a href="https://cicl.stanford.edu/publication/jin2024marple/" itemprop="name">MARPLE: A Benchmark for Long-Horizon Inference</a>.
+  <em>Advances in Neural Information Processing Systems</em>.
+
+
+
+
+  <p>
+
+
+
+
+<a class="btn btn-outline-primary my-1 mr-1 btn-sm" href="http://arxiv.org/abs/2410.01926" target="_blank" rel="noopener">
+  Preprint
+</a>
+
+
+<a class="btn btn-outline-primary my-1 mr-1 btn-sm" href="https://cicl.stanford.edu/papers/jin2024marple.pdf" target="_blank" rel="noopener">
+  PDF
+</a>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<a class="btn btn-outline-primary my-1 mr-1 btn-sm" href="https://marple-benchmark.github.io/" target="_blank" rel="noopener">
+  Project website
+</a>
+
+
+  </p>
+
+</div>
+<div class="pub-list-item" style="margin-bottom: 1rem" itemscope itemtype="http://schema.org/CreativeWork">
   <span itemprop="author">
     A. Beller, T. Gerstenberg</span>
 
@@ -420,6 +467,10 @@ <h2 id="publications">Publications</h2>
 </a>
 
 
+<a class="btn btn-outline-primary my-1 mr-1 btn-sm" href="https://cicl.stanford.edu/papers/gandhi2024affective.pdf" target="_blank" rel="noopener">
+  PDF
+</a>
+
 
 
 
@@ -926,7 +977,7 @@ <h2 id="publications">Publications</h2>
     (2024).
 
   <a href="https://cicl.stanford.edu/publication/franken2024sami/" itemprop="name">Self-supervised alignment with mutual information: Learning to follow principles without preference labels</a>.
-  <em>arXiv</em>.
+  <em>Advances in Neural Information Processing Systems</em>.
 
 
 

docs/publication/franken2024sami/index.html

@@ -266,8 +266,8 @@ <h3>Abstract</h3>
           <div class="col-xs-12 col-sm-3 pub-row-heading">Type</div>
           <div class="col-xs-12 col-sm-9">
 
-            <a href="/publication/#1">
-              Preprint
+            <a href="/publication/#3">
+              Conference Proceedings
             </a>
 
           </div>
@@ -283,7 +283,7 @@ <h3>Abstract</h3>
       <div class="col-sm-10">
         <div class="row">
           <div class="col-xs-12 col-sm-3 pub-row-heading">Publication</div>
-          <div class="col-xs-12 col-sm-9">Fränken, J., Zelikman, E., Rafailov, R., Gandhi, K., Gerstenberg, T., Goodman, N. D. (2024). Self-supervised alignment with mutual information: Learning to follow principles without preference labels. <em>arXiv</em>.</div>
+          <div class="col-xs-12 col-sm-9">Fränken, J., Zelikman, E., Rafailov, R., Gandhi, K., Gerstenberg, T., Goodman, N. D. (2024). Self-supervised alignment with mutual information: Learning to follow principles without preference labels. <em>Advances in Neural Information Processing Systems</em>.</div>
         </div>
       </div>
       <div class="col-sm-1"></div>

docs/publication/gandhi2024affective/index.html

@@ -320,6 +320,10 @@ <h3>Abstract</h3>
 </a>
 
 
+<a class="btn btn-outline-primary my-1 mr-1" href="https://cicl.stanford.edu/papers/gandhi2024affective.pdf" target="_blank" rel="noopener">
+  PDF
+</a>
+
 
 
 

docs/publication/index.html

@@ -1597,6 +1597,19 @@ <h1>Publications</h1>
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -1733,6 +1746,65 @@ <h1>Publications</h1>
 
 
 
+				<div class='grid-sizer col-md-12 isotope-item pubtype-3 year-2024 author-'>
+
+						<div class="pub-list-item" style="margin-bottom: 1rem" itemscope itemtype="http://schema.org/CreativeWork">
+  <span itemprop="author">
+    E. Jin, Z. Huang, J. Fränken, W. Liu, H. Cha, E. Brockbank, S. Wu, R. Zhang, J. Wu, T. Gerstenberg</span>
+
+    (2024).
+
+  <a href="https://cicl.stanford.edu/publication/jin2024marple/" itemprop="name">MARPLE: A Benchmark for Long-Horizon Inference</a>.
+  <em>Advances in Neural Information Processing Systems</em>.
+
+
+
+
+  <p>
+
+
+
+
+<a class="btn btn-outline-primary my-1 mr-1 btn-sm" href="http://arxiv.org/abs/2410.01926" target="_blank" rel="noopener">
+  Preprint
+</a>
+
+
+<a class="btn btn-outline-primary my-1 mr-1 btn-sm" href="https://cicl.stanford.edu/papers/jin2024marple.pdf" target="_blank" rel="noopener">
+  PDF
+</a>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<a class="btn btn-outline-primary my-1 mr-1 btn-sm" href="https://marple-benchmark.github.io/" target="_blank" rel="noopener">
+  Project website
+</a>
+
+
+  </p>
+
+</div>
+
+
+				</div>
+
+
+
+
+
+
+
 				<div class='grid-sizer col-md-12 isotope-item pubtype-1 year-2024 author-'>
 
 						<div class="pub-list-item" style="margin-bottom: 1rem" itemscope itemtype="http://schema.org/CreativeWork">
@@ -1812,6 +1884,10 @@ <h1>Publications</h1>
 </a>
 
 
+<a class="btn btn-outline-primary my-1 mr-1 btn-sm" href="https://cicl.stanford.edu/papers/gandhi2024affective.pdf" target="_blank" rel="noopener">
+  PDF
+</a>
+
 
 
 
@@ -2441,7 +2517,7 @@ <h1>Publications</h1>
 
 
 
-				<div class='grid-sizer col-md-12 isotope-item pubtype-1 year-2024 author-'>
+				<div class='grid-sizer col-md-12 isotope-item pubtype-3 year-2024 author-'>
 
 						<div class="pub-list-item" style="margin-bottom: 1rem" itemscope itemtype="http://schema.org/CreativeWork">
   <span itemprop="author">
@@ -2450,7 +2526,7 @@ <h1>Publications</h1>
     (2024).
 
   <a href="https://cicl.stanford.edu/publication/franken2024sami/" itemprop="name">Self-supervised alignment with mutual information: Learning to follow principles without preference labels</a>.
-  <em>arXiv</em>.
+  <em>Advances in Neural Information Processing Systems</em>.
 
 
 

docs/publication/index.xml

@@ -12,6 +12,15 @@
 	<atom:link href="https://cicl.stanford.edu/publication/index.xml" rel="self" type="application/rss+xml" />
 
 
+    <item>
+      <title>MARPLE: A Benchmark for Long-Horizon Inference</title>
+      <link>https://cicl.stanford.edu/publication/jin2024marple/</link>
+      <pubDate>Fri, 04 Oct 2024 00:00:00 +0000</pubDate>
+
+      <guid>https://cicl.stanford.edu/publication/jin2024marple/</guid>
+      <description></description>
+    </item>
+
     <item>
       <title>Causation, Meaning, and Communication</title>
       <link>https://cicl.stanford.edu/publication/beller2024causation/</link>