@article{2404.14219v4, Author        = {Marah Abdin and Jyoti Aneja and Hany Awadalla and Ahmed Awadallah and Ammar Ahmad Awan and Nguyen Bach and Amit Bahree and Arash Bakhtiari and Jianmin Bao and Harkirat Behl and Alon Benhaim and Misha Bilenko and Johan Bjorck and Sébastien Bubeck and Martin Cai and Qin Cai and Vishrav Chaudhary and Dong Chen and Dongdong Chen and Weizhu Chen and Yen-Chun Chen and Yi-Ling Chen and Hao Cheng and Parul Chopra and Xiyang Dai and Matthew Dixon and Ronen Eldan and Victor Fragoso and Jianfeng Gao and Mei Gao and Min Gao and Amit Garg and Allie Del Giorno and Abhishek Goswami and Suriya Gunasekar and Emman Haider and Junheng Hao and Russell J. Hewett and Wenxiang Hu and Jamie Huynh and Dan Iter and Sam Ade Jacobs and Mojan Javaheripi and Xin Jin and Nikos Karampatziakis and Piero Kauffmann and Mahoud Khademi and Dongwoo Kim and Young Jin Kim and Lev Kurilenko and James R. Lee and Yin Tat Lee and Yuanzhi Li and Yunsheng Li and Chen Liang and Lars Liden and Xihui Lin and Zeqi Lin and Ce Liu and Liyuan Liu and Mengchen Liu and Weishung Liu and Xiaodong Liu and Chong Luo and Piyush Madan and Ali Mahmoudzadeh and David Majercak and Matt Mazzola and Caio César Teodoro Mendes and Arindam Mitra and Hardik Modi and Anh Nguyen and Brandon Norick and Barun Patra and Daniel Perez-Becker and Thomas Portet and Reid Pryzant and Heyang Qin and Marko Radmilac and Liliang Ren and Gustavo de Rosa and Corby Rosset and Sambudha Roy and Olatunji Ruwase and Olli Saarikivi and Amin Saied and Adil Salim and Michael Santacroce and Shital Shah and Ning Shang and Hiteshi Sharma and Yelong Shen and Swadheen Shukla and Xia Song and Masahiro Tanaka and Andrea Tupini and Praneetha Vaddamanu and Chunyu Wang and Guanhua Wang and Lijuan Wang and Shuohang Wang and Xin Wang and Yu Wang and Rachel Ward and Wen Wen and Philipp Witte and Haiping Wu and Xiaoxia Wu and Michael Wyatt and Bin Xiao and Can Xu and Jiahang Xu and Weijian Xu and Jilong Xue and Sonali Yadav and Fan Yang and Jianwei Yang and Yifan Yang and Ziyi Yang and Donghan Yu and Lu Yuan and Chenruidong Zhang and Cyril Zhang and Jianwen Zhang and Li Lyna Zhang and Yi Zhang and Yue Zhang and Yunan Zhang and Xiren Zhou}, Title         = {Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone}, Eprint        = {http://arxiv.org/abs/2404.14219v4}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {We introduce phi-3-mini, a 3.8 billion parameter language model trained on 3.3 trillion tokens, whose overall performance, as measured by both academic benchmarks and internal testing, rivals that of models such as Mixtral 8x7B and GPT-3.5 (e.g., phi-3-mini achieves 69% on MMLU and 8.38 on MT-bench), despite being small enough to be deployed on a phone. Our training dataset is a scaled-up version of the one used for phi-2, composed of heavily filtered publicly available web data and synthetic data. The model is also further aligned for robustness, safety, and chat format. We also provide parameter-scaling results with a 7B, 14B models trained for 4.8T tokens, called phi-3-small, phi-3-medium, both significantly more capable than phi-3-mini (e.g., respectively 75%, 78% on MMLU, and 8.7, 8.9 on MT-bench). To enhance multilingual, multimodal, and long-context capabilities, we introduce three models in the phi-3.5 series: phi-3.5-mini, phi-3.5-MoE, and phi-3.5-Vision. The phi-3.5-MoE, a 16 x 3.8B MoE model with 6.6 billion active parameters, achieves superior performance in language reasoning, math, and code tasks compared to other open-source models of similar scale, such as Llama 3.1 and the Mixtral series, and on par with Gemini-1.5-Flash and GPT-4o-mini. Meanwhile, phi-3.5-Vision, a 4.2 billion parameter model derived from phi-3.5-mini, excels in reasoning tasks and is adept at handling both single-image and text prompts, as well as multi-image and text prompts.}, Year          = {2024}, Month         = {4}, Url           = {http://arxiv.org/pdf/2404.14219v4}, File          = {2404.14219v4.pdf} }
 
@article{2405.03548v4, Author        = {Xiang Yue and Tuney Zheng and Ge Zhang and Wenhu Chen}, Title         = {MAmmoTH2: Scaling Instructions from the Web}, Eprint        = {http://arxiv.org/abs/2405.03548v4}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Instruction tuning improves the reasoning abilities of large language models (LLMs), with data quality and scalability being the crucial factors. Most instruction tuning data come from human crowd-sourcing or GPT-4 distillation. We propose a paradigm to efficiently harvest 10 million naturally existing instruction data from the pre-training web corpus to enhance LLM reasoning. Our approach involves (1) recalling relevant documents, (2) extracting instruction-response pairs, and (3) refining the extracted pairs using open-source LLMs. Fine-tuning base LLMs on this dataset, we build MAmmoTH2 models, which significantly boost performance on reasoning benchmarks. Notably, MAmmoTH2-7B's (Mistral) performance increases from 11% to 36.7% on MATH and from 36% to 68.4% on GSM8K without training on any in-domain data. Further training MAmmoTH2 on public instruction tuning datasets yields MAmmoTH2-Plus, achieving state-of-the-art performance on several reasoning and chatbot benchmarks. Our work demonstrates how to harvest large-scale, high-quality instruction data without costly human annotation or GPT-4 distillation, providing a new paradigm for building better instruction tuning data.}, Year          = {2024}, Month         = {5}, Url           = {http://arxiv.org/pdf/2405.03548v4}, File          = {2405.03548v4.pdf} }
 
@article{2309.05653v3, Author        = {Xiang Yue and Xingwei Qu and Ge Zhang and Yao Fu and Wenhao Huang and Huan Sun and Yu Su and Wenhu Chen}, Title         = {MAmmoTH: Building Math Generalist Models through Hybrid Instruction Tuning}, Eprint        = {http://arxiv.org/abs/2309.05653v3}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {We introduce MAmmoTH, a series of open-source large language models (LLMs) specifically tailored for general math problem-solving. The MAmmoTH models are trained on MathInstruct, our meticulously curated instruction tuning dataset. MathInstruct is compiled from 13 math datasets with intermediate rationales, six of which have rationales newly curated by us. It presents a unique hybrid of chain-of-thought (CoT) and program-of-thought (PoT) rationales, and also ensures extensive coverage of diverse fields in math. The hybrid of CoT and PoT not only unleashes the potential of tool use but also allows different thought processes for different math problems. As a result, the MAmmoTH series substantially outperform existing open-source models on nine mathematical reasoning datasets across all scales with an average accuracy gain between 16% and 32%. Remarkably, our MAmmoTH-7B model reaches 33% on MATH (a competition-level dataset), which exceeds the best open-source 7B model (WizardMath) by 23%, and the MAmmoTH-34B model achieves 44% accuracy on MATH, even surpassing GPT-4's CoT result. Our work underscores the importance of diverse problem coverage and the use of hybrid rationales in developing superior math generalist models.}, Year          = {2023}, Month         = {9}, Url           = {http://arxiv.org/pdf/2309.05653v3}, File          = {2309.05653v3.pdf} }
 
@article{2405.05904v3, Author        = {Zorik Gekhman and Gal Yona and Roee Aharoni and Matan Eyal and Amir Feder and Roi Reichart and Jonathan Herzig}, Title         = {Does Fine-Tuning LLMs on New Knowledge Encourage Hallucinations?}, Eprint        = {http://arxiv.org/abs/2405.05904v3}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {When large language models are aligned via supervised fine-tuning, they may encounter new factual information that was not acquired through pre-training. It is often conjectured that this can teach the model the behavior of hallucinating factually incorrect responses, as the model is trained to generate facts that are not grounded in its pre-existing knowledge. In this work, we study the impact of such exposure to new knowledge on the capability of the fine-tuned model to utilize its pre-existing knowledge. To this end, we design a controlled setup, focused on closed-book QA, where we vary the proportion of the fine-tuning examples that introduce new knowledge. We demonstrate that large language models struggle to acquire new factual knowledge through fine-tuning, as fine-tuning examples that introduce new knowledge are learned significantly slower than those consistent with the model's knowledge. However, we also find that as the examples with new knowledge are eventually learned, they linearly increase the model's tendency to hallucinate. Taken together, our results highlight the risk in introducing new factual knowledge through fine-tuning, and support the view that large language models mostly acquire factual knowledge through pre-training, whereas fine-tuning teaches them to use it more efficiently.}, Year          = {2024}, Month         = {5}, Url           = {http://arxiv.org/pdf/2405.05904v3}, File          = {2405.05904v3.pdf} }
 
@article{2203.02155v1, Author        = {Long Ouyang and Jeff Wu and Xu Jiang and Diogo Almeida and Carroll L. Wainwright and Pamela Mishkin and Chong Zhang and Sandhini Agarwal and Katarina Slama and Alex Ray and John Schulman and Jacob Hilton and Fraser Kelton and Luke Miller and Maddie Simens and Amanda Askell and Peter Welinder and Paul Christiano and Jan Leike and Ryan Lowe}, Title         = {Training language models to follow instructions with human feedback}, Eprint        = {http://arxiv.org/abs/2203.02155v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Making language models bigger does not inherently make them better at following a user's intent. For example, large language models can generate outputs that are untruthful, toxic, or simply not helpful to the user. In other words, these models are not aligned with their users. In this paper, we show an avenue for aligning language models with user intent on a wide range of tasks by fine-tuning with human feedback. Starting with a set of labeler-written prompts and prompts submitted through the OpenAI API, we collect a dataset of labeler demonstrations of the desired model behavior, which we use to fine-tune GPT-3 using supervised learning. We then collect a dataset of rankings of model outputs, which we use to further fine-tune this supervised model using reinforcement learning from human feedback. We call the resulting models InstructGPT. In human evaluations on our prompt distribution, outputs from the 1.3B parameter InstructGPT model are preferred to outputs from the 175B GPT-3, despite having 100x fewer parameters. Moreover, InstructGPT models show improvements in truthfulness and reductions in toxic output generation while having minimal performance regressions on public NLP datasets. Even though InstructGPT still makes simple mistakes, our results show that fine-tuning with human feedback is a promising direction for aligning language models with human intent.}, Year          = {2022}, Month         = {3}, Url           = {http://arxiv.org/pdf/2203.02155v1}, File          = {2203.02155v1.pdf} }
 
@article{2305.18290v3, Author        = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Stefano Ermon and Christopher D. Manning and Chelsea Finn}, Title         = {Direct Preference Optimization: Your Language Model is Secretly a Reward Model}, Eprint        = {http://arxiv.org/abs/2305.18290v3}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.LG}, Abstract      = {While large-scale unsupervised language models (LMs) learn broad world knowledge and some reasoning skills, achieving precise control of their behavior is difficult due to the completely unsupervised nature of their training. Existing methods for gaining such steerability collect human labels of the relative quality of model generations and fine-tune the unsupervised LM to align with these preferences, often with reinforcement learning from human feedback (RLHF). However, RLHF is a complex and often unstable procedure, first fitting a reward model that reflects the human preferences, and then fine-tuning the large unsupervised LM using reinforcement learning to maximize this estimated reward without drifting too far from the original model. In this paper we introduce a new parameterization of the reward model in RLHF that enables extraction of the corresponding optimal policy in closed form, allowing us to solve the standard RLHF problem with only a simple classification loss. The resulting algorithm, which we call Direct Preference Optimization (DPO), is stable, performant, and computationally lightweight, eliminating the need for sampling from the LM during fine-tuning or performing significant hyperparameter tuning. Our experiments show that DPO can fine-tune LMs to align with human preferences as well as or better than existing methods. Notably, fine-tuning with DPO exceeds PPO-based RLHF in ability to control sentiment of generations, and matches or improves response quality in summarization and single-turn dialogue while being substantially simpler to implement and train.}, Year          = {2023}, Month         = {5}, Url           = {http://arxiv.org/pdf/2305.18290v3}, File          = {2305.18290v3.pdf} }
 
@article{2303.17651v2, Author        = {Aman Madaan and Niket Tandon and Prakhar Gupta and Skyler Hallinan and Luyu Gao and Sarah Wiegreffe and Uri Alon and Nouha Dziri and Shrimai Prabhumoye and Yiming Yang and Shashank Gupta and Bodhisattwa Prasad Majumder and Katherine Hermann and Sean Welleck and Amir Yazdanbakhsh and Peter Clark}, Title         = {Self-Refine: Iterative Refinement with Self-Feedback}, Eprint        = {http://arxiv.org/abs/2303.17651v2}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Like humans, large language models (LLMs) do not always generate the best output on their first try. Motivated by how humans refine their written text, we introduce Self-Refine, an approach for improving initial outputs from LLMs through iterative feedback and refinement. The main idea is to generate an initial output using an LLMs; then, the same LLMs provides feedback for its output and uses it to refine itself, iteratively. Self-Refine does not require any supervised training data, additional training, or reinforcement learning, and instead uses a single LLM as the generator, refiner, and feedback provider. We evaluate Self-Refine across 7 diverse tasks, ranging from dialog response generation to mathematical reasoning, using state-of-the-art (GPT-3.5, ChatGPT, and GPT-4) LLMs. Across all evaluated tasks, outputs generated with Self-Refine are preferred by humans and automatic metrics over those generated with the same LLM using conventional one-step generation, improving by ~20% absolute on average in task performance. Our work demonstrates that even state-of-the-art LLMs like GPT-4 can be further improved at test time using our simple, standalone approach.}, Year          = {2023}, Month         = {3}, Url           = {http://arxiv.org/pdf/2303.17651v2}, File          = {2303.17651v2.pdf} }
 
@article{2208.03306v1, Author        = {Margaret Li and Suchin Gururangan and Tim Dettmers and Mike Lewis and Tim Althoff and Noah A. Smith and Luke Zettlemoyer}, Title         = {Branch-Train-Merge: Embarrassingly Parallel Training of Expert Language Models}, Eprint        = {http://arxiv.org/abs/2208.03306v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {We present Branch-Train-Merge (BTM), a communication-efficient algorithm for embarrassingly parallel training of large language models (LLMs). We show it is possible to independently train subparts of a new class of LLMs on different subsets of the data, eliminating the massive multi-node synchronization currently required to train LLMs. BTM learns a set of independent expert LMs (ELMs), each specialized to a different textual domain, such as scientific or legal text. These ELMs can be added and removed to update data coverage, ensembled to generalize to new domains, or averaged to collapse back to a single LM for efficient inference. New ELMs are learned by branching from (mixtures of) ELMs in the current set, further training the parameters on data for the new domain, and then merging the resulting model back into the set for future use. Experiments show that BTM improves in- and out-of-domain perplexities as compared to GPT-style Transformer LMs, when controlling for training cost. Through extensive analysis, we show that these results are robust to different ELM initialization schemes, but require expert domain specialization; LM ensembles with random data splits do not perform well. We also present a study of scaling BTM into a new corpus of 64 domains (192B whitespace-separated tokens in total); the resulting LM (22.4B total parameters) performs as well as a Transformer LM trained with 2.5 times more compute. These gains grow with the number of domains, suggesting more aggressive parallelism could be used to efficiently train larger models in future work.}, Year          = {2022}, Month         = {8}, Url           = {http://arxiv.org/pdf/2208.03306v1}, File          = {2208.03306v1.pdf} }
 
@article{2402.13228v2, Author        = {Arka Pal and Deep Karkhanis and Samuel Dooley and Manley Roberts and Siddartha Naidu and Colin White}, Title         = {Smaug: Fixing Failure Modes of Preference Optimisation with DPO-Positive}, Eprint        = {http://arxiv.org/abs/2402.13228v2}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Direct Preference Optimisation (DPO) is effective at significantly improving the performance of large language models (LLMs) on downstream tasks such as reasoning, summarisation, and alignment. Using pairs of preferred and dispreferred data, DPO models the relative probability of picking one response over another. In this work, first we show theoretically that the standard DPO loss can lead to a reduction of the model's likelihood of the preferred examples, as long as the relative probability between the preferred and dispreferred classes increases. We then show empirically that this phenomenon occurs when fine-tuning LLMs on common datasets, especially datasets in which the edit distance between pairs of completions is low. Using these insights, we design DPO-Positive (DPOP), a new loss function and training procedure which avoids this failure mode. Surprisingly, we find that DPOP outperforms DPO and other fine-tuning procedures across a wide variety of datasets and downstream tasks, including datasets with high edit distances between completions. Furthermore, we find that the DPOP-tuned model outperforms the DPO-tuned model (all else equal) on benchmarks independent of the fine-tuning data, such as MT-Bench. Finally, using DPOP, we create and open-source Smaug-34B and Smaug-72B, with the latter becoming the first open-source LLM to surpass an average accuracy of 80% on the HuggingFace Open LLM Leaderboard.}, Year          = {2024}, Month         = {2}, Url           = {http://arxiv.org/pdf/2402.13228v2}, File          = {2402.13228v2.pdf} }
 
@article{2310.01889v4, Author        = {Hao Liu and Matei Zaharia and Pieter Abbeel}, Title         = {Ring Attention with Blockwise Transformers for Near-Infinite Context}, Eprint        = {http://arxiv.org/abs/2310.01889v4}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Transformers have emerged as the architecture of choice for many state-of-the-art AI models, showcasing exceptional performance across a wide range of AI applications. However, the memory demands imposed by Transformers limit their ability to handle long sequences, thereby posing challenges in utilizing videos, actions, and other long-form sequences and modalities in complex environments. We present a novel approach, Ring Attention with Blockwise Transformers (Ring Attention), which leverages blockwise computation of self-attention and feedforward to distribute long sequences across multiple devices while fully overlapping the communication of key-value blocks with the computation of blockwise attention. Our approach enables training and inference of sequences that are up to device count times longer than those achievable by prior memory-efficient Transformers, without resorting to approximations or incurring additional communication and computation overheads. Extensive experiments on language modeling and reinforcement learning tasks demonstrate the effectiveness of our approach in allowing millions of tokens context size and improving performance.}, Year          = {2023}, Month         = {10}, Url           = {http://arxiv.org/pdf/2310.01889v4}, File          = {2310.01889v4.pdf} }
 
@article{2210.03057v1, Author        = {Freda Shi and Mirac Suzgun and Markus Freitag and Xuezhi Wang and Suraj Srivats and Soroush Vosoughi and Hyung Won Chung and Yi Tay and Sebastian Ruder and Denny Zhou and Dipanjan Das and Jason Wei}, Title         = {Language Models are Multilingual Chain-of-Thought Reasoners}, Eprint        = {http://arxiv.org/abs/2210.03057v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {We evaluate the reasoning abilities of large language models in multilingual settings. We introduce the Multilingual Grade School Math (MGSM) benchmark, by manually translating 250 grade-school math problems from the GSM8K dataset (Cobbe et al., 2021) into ten typologically diverse languages. We find that the ability to solve MGSM problems via chain-of-thought prompting emerges with increasing model scale, and that models have strikingly strong multilingual reasoning abilities, even in underrepresented languages such as Bengali and Swahili. Finally, we show that the multilingual reasoning abilities of language models extend to other tasks such as commonsense reasoning and word-in-context semantic judgment. The MGSM benchmark is publicly available at https://github.com/google-research/url-nlp.}, Year          = {2022}, Month         = {10}, Url           = {http://arxiv.org/pdf/2210.03057v1}, File          = {2210.03057v1.pdf} }
 
@article{2210.09261v1, Author        = {Mirac Suzgun and Nathan Scales and Nathanael Schärli and Sebastian Gehrmann and Yi Tay and Hyung Won Chung and Aakanksha Chowdhery and Quoc V. Le and Ed H. Chi and Denny Zhou and Jason Wei}, Title         = {Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them}, Eprint        = {http://arxiv.org/abs/2210.09261v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {BIG-Bench (Srivastava et al., 2022) is a diverse evaluation suite that focuses on tasks believed to be beyond the capabilities of current language models. Language models have already made good progress on this benchmark, with the best model in the BIG-Bench paper outperforming average reported human-rater results on 65% of the BIG-Bench tasks via few-shot prompting. But on what tasks do language models fall short of average human-rater performance, and are those tasks actually unsolvable by current language models?   In this work, we focus on a suite of 23 challenging BIG-Bench tasks which we call BIG-Bench Hard (BBH). These are the task for which prior language model evaluations did not outperform the average human-rater. We find that applying chain-of-thought (CoT) prompting to BBH tasks enables PaLM to surpass the average human-rater performance on 10 of the 23 tasks, and Codex (code-davinci-002) to surpass the average human-rater performance on 17 of the 23 tasks. Since many tasks in BBH require multi-step reasoning, few-shot prompting without CoT, as done in the BIG-Bench evaluations (Srivastava et al., 2022), substantially underestimates the best performance and capabilities of language models, which is better captured via CoT prompting. As further analysis, we explore the interaction between CoT and model scale on BBH, finding that CoT enables emergent task performance on several BBH tasks with otherwise flat scaling curves.}, Year          = {2022}, Month         = {10}, Url           = {http://arxiv.org/pdf/2210.09261v1}, File          = {2210.09261v1.pdf} }
 
@article{2403.04706v1, Author        = {Chen Li and Weiqi Wang and Jingcheng Hu and Yixuan Wei and Nanning Zheng and Han Hu and Zheng Zhang and Houwen Peng}, Title         = {Common 7B Language Models Already Possess Strong Math Capabilities}, Eprint        = {http://arxiv.org/abs/2403.04706v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Mathematical capabilities were previously believed to emerge in common language models only at a very large scale or require extensive math-related pre-training. This paper shows that the LLaMA-2 7B model with common pre-training already exhibits strong mathematical abilities, as evidenced by its impressive accuracy of 97.7% and 72.0% on the GSM8K and MATH benchmarks, respectively, when selecting the best response from 256 random generations. The primary issue with the current base model is the difficulty in consistently eliciting its inherent mathematical capabilities. Notably, the accuracy for the first answer drops to 49.5% and 7.9% on the GSM8K and MATH benchmarks, respectively. We find that simply scaling up the SFT data can significantly enhance the reliability of generating correct answers. However, the potential for extensive scaling is constrained by the scarcity of publicly available math questions. To overcome this limitation, we employ synthetic data, which proves to be nearly as effective as real data and shows no clear saturation when scaled up to approximately one million samples. This straightforward approach achieves an accuracy of 82.6% on GSM8K and 40.6% on MATH using LLaMA-2 7B models, surpassing previous models by 14.2% and 20.8%, respectively. We also provide insights into scaling behaviors across different reasoning complexities and error types.}, Year          = {2024}, Month         = {3}, Url           = {http://arxiv.org/pdf/2403.04706v1}, File          = {2403.04706v1.pdf} }
 
@article{2405.12205v1, Author        = {Aniket Didolkar and Anirudh Goyal and Nan Rosemary Ke and Siyuan Guo and Michal Valko and Timothy Lillicrap and Danilo Rezende and Yoshua Bengio and Michael Mozer and Sanjeev Arora}, Title         = {Metacognitive Capabilities of LLMs: An Exploration in Mathematical Problem Solving}, Eprint        = {http://arxiv.org/abs/2405.12205v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.AI}, Abstract      = {Metacognitive knowledge refers to humans' intuitive knowledge of their own thinking and reasoning processes. Today's best LLMs clearly possess some reasoning processes. The paper gives evidence that they also have metacognitive knowledge, including ability to name skills and procedures to apply given a task. We explore this primarily in context of math reasoning, developing a prompt-guided interaction procedure to get a powerful LLM to assign sensible skill labels to math questions, followed by having it perform semantic clustering to obtain coarser families of skill labels. These coarse skill labels look interpretable to humans.   To validate that these skill labels are meaningful and relevant to the LLM's reasoning processes we perform the following experiments. (a) We ask GPT-4 to assign skill labels to training questions in math datasets GSM8K and MATH. (b) When using an LLM to solve the test questions, we present it with the full list of skill labels and ask it to identify the skill needed. Then it is presented with randomly selected exemplar solved questions associated with that skill label. This improves accuracy on GSM8k and MATH for several strong LLMs, including code-assisted models. The methodology presented is domain-agnostic, even though this article applies it to math problems.}, Year          = {2024}, Month         = {5}, Url           = {http://arxiv.org/pdf/2405.12205v1}, File          = {2405.12205v1.pdf} }
 
@article{2402.19255v2, Author        = {Qintong Li and Leyang Cui and Xueliang Zhao and Lingpeng Kong and Wei Bi}, Title         = {GSM-Plus: A Comprehensive Benchmark for Evaluating the Robustness of LLMs as Mathematical Problem Solvers}, Eprint        = {http://arxiv.org/abs/2402.19255v2}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Large language models (LLMs) have achieved impressive performance across various mathematical reasoning benchmarks. However, there are increasing debates regarding whether these models truly understand and apply mathematical knowledge or merely rely on shortcuts for mathematical reasoning. One essential and frequently occurring evidence is that when the math questions are slightly changed, LLMs can behave incorrectly. This motivates us to evaluate the robustness of LLMs' math reasoning capability by testing a wide range of question variations. We introduce the adversarial grade school math (GSM-Plus) dataset, an extension of GSM8K augmented with various mathematical perturbations. Our experiments on 25 LLMs and 4 prompting techniques show that while LLMs exhibit different levels of math reasoning abilities, their performances are far from robust. In particular, even for problems that have been solved in GSM8K, LLMs can make mistakes when new statements are added or the question targets are altered. We also explore whether more robust performance can be achieved by composing existing prompting methods, in which we try an iterative method that generates and verifies each intermediate thought based on its reasoning goal and calculation result.}, Year          = {2024}, Month         = {2}, Url           = {http://arxiv.org/pdf/2402.19255v2}, File          = {2402.19255v2.pdf} }
 
@article{2210.03629v3, Author        = {Shunyu Yao and Jeffrey Zhao and Dian Yu and Nan Du and Izhak Shafran and Karthik Narasimhan and Yuan Cao}, Title         = {ReAct: Synergizing Reasoning and Acting in Language Models}, Eprint        = {http://arxiv.org/abs/2210.03629v3}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {While large language models (LLMs) have demonstrated impressive capabilities across tasks in language understanding and interactive decision making, their abilities for reasoning (e.g. chain-of-thought prompting) and acting (e.g. action plan generation) have primarily been studied as separate topics. In this paper, we explore the use of LLMs to generate both reasoning traces and task-specific actions in an interleaved manner, allowing for greater synergy between the two: reasoning traces help the model induce, track, and update action plans as well as handle exceptions, while actions allow it to interface with external sources, such as knowledge bases or environments, to gather additional information. We apply our approach, named ReAct, to a diverse set of language and decision making tasks and demonstrate its effectiveness over state-of-the-art baselines, as well as improved human interpretability and trustworthiness over methods without reasoning or acting components. Concretely, on question answering (HotpotQA) and fact verification (Fever), ReAct overcomes issues of hallucination and error propagation prevalent in chain-of-thought reasoning by interacting with a simple Wikipedia API, and generates human-like task-solving trajectories that are more interpretable than baselines without reasoning traces. On two interactive decision making benchmarks (ALFWorld and WebShop), ReAct outperforms imitation and reinforcement learning methods by an absolute success rate of 34% and 10% respectively, while being prompted with only one or two in-context examples. Project site with code: https://react-lm.github.io}, Year          = {2022}, Month         = {10}, Url           = {http://arxiv.org/pdf/2210.03629v3}, File          = {2210.03629v3.pdf} }
 
@article{2309.17452v4, Author        = {Zhibin Gou and Zhihong Shao and Yeyun Gong and Yelong Shen and Yujiu Yang and Minlie Huang and Nan Duan and Weizhu Chen}, Title         = {ToRA: A Tool-Integrated Reasoning Agent for Mathematical Problem Solving}, Eprint        = {http://arxiv.org/abs/2309.17452v4}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Large language models have made significant progress in various language tasks, yet they still struggle with complex mathematics. In this paper, we propose ToRA a series of Tool-integrated Reasoning Agents designed to solve challenging mathematical problems by seamlessly integrating natural language reasoning with the utilization of external tools (e.g., computation libraries and symbolic solvers), thereby amalgamating the analytical prowess of language and the computational efficiency of tools. To train ToRA, we curate interactive tool-use trajectories on mathematical datasets, apply imitation learning on the annotations, and propose output space shaping to further refine models' reasoning behavior. As a result, ToRA models significantly outperform open-source models on 10 mathematical reasoning datasets across all scales with 13%-19% absolute improvements on average. Notably, ToRA-7B reaches 44.6% on the competition-level dataset MATH, surpassing the best open-source model WizardMath-70B by 22% absolute. ToRA-Code-34B is also the first open-source model that achieves an accuracy exceeding 50% on MATH, which significantly outperforms GPT-4's CoT result, and is competitive with GPT-4 solving problems with programs. Additionally, we conduct a comprehensive analysis of the benefits and remaining challenges of tool interaction for mathematical reasoning, providing valuable insights for future research.}, Year          = {2023}, Month         = {9}, Url           = {http://arxiv.org/pdf/2309.17452v4}, File          = {2309.17452v4.pdf} }
 
@article{2309.12284v4, Author        = {Longhui Yu and Weisen Jiang and Han Shi and Jincheng Yu and Zhengying Liu and Yu Zhang and James T. Kwok and Zhenguo Li and Adrian Weller and Weiyang Liu}, Title         = {MetaMath: Bootstrap Your Own Mathematical Questions for Large Language Models}, Eprint        = {http://arxiv.org/abs/2309.12284v4}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Large language models (LLMs) have pushed the limits of natural language understanding and exhibited excellent problem-solving ability. Despite the great success, most existing open-source LLMs (e.g., LLaMA-2) are still far away from satisfactory for solving mathematical problem due to the complex reasoning procedures. To bridge this gap, we propose MetaMath, a fine-tuned language model that specializes in mathematical reasoning. Specifically, we start by bootstrapping mathematical questions by rewriting the question from multiple perspectives without extra knowledge, which results in a new dataset called MetaMathQA. Then we fine-tune the LLaMA-2 models on MetaMathQA. Experimental results on two popular benchmarks (i.e., GSM8K and MATH) for mathematical reasoning demonstrate that MetaMath outperforms a suite of open-source LLMs by a significant margin. Our MetaMath-7B model achieves 66.4% on GSM8K and 19.4% on MATH, exceeding the state-of-the-art models of the same size by 11.5% and 8.7%. Particularly, MetaMath-70B achieves an accuracy of 82.3% on GSM8K, slightly better than GPT-3.5-Turbo. We release all the MetaMathQA dataset, the MetaMath models with different model sizes and the training code for public use.}, Year          = {2023}, Month         = {9}, Url           = {http://arxiv.org/pdf/2309.12284v4}, File          = {2309.12284v4.pdf} }
 
@article{2310.20246v5, Author        = {Nuo Chen and Zinan Zheng and Ning Wu and Ming Gong and Dongmei Zhang and Jia Li}, Title         = {Breaking Language Barriers in Multilingual Mathematical Reasoning: Insights and Observations}, Eprint        = {http://arxiv.org/abs/2310.20246v5}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Existing research predominantly focuses on developing powerful language learning models (LLMs) for mathematical reasoning within monolingual languages, with few explorations in preserving efficacy in a multilingual context. To bridge this gap, this paper pioneers exploring and training powerful Multilingual Math Reasoning (xMR) LLMs. Firstly, by utilizing translation, we construct the first multilingual math reasoning instruction dataset, MGSM8KInstruct, encompassing ten distinct languages, thus addressing the issue of training data scarcity in xMR tasks. Based on the collected dataset, we propose different training strategies to build powerful xMR LLMs, named MathOctopus, notably outperform conventional open-source LLMs and exhibit superiority over ChatGPT in few-shot scenarios. Notably, MathOctopus-13B reaches 47.6% accuracy which exceeds ChatGPT 46.3% on MGSM testset. Beyond remarkable results, we unearth several pivotal observations and insights from extensive experiments: (1) When extending the rejection sampling strategy to the multilingual context, it proves effective for model performances, albeit limited. (2) Employing parallel corpora for math Supervised Fine-Tuning (SFT) across multiple languages not only significantly enhances model performance multilingually but also elevates their monolingual performance. This indicates that crafting multilingual corpora can be regarded as a vital strategy for enhancing model performance in a specific language, especially in mathematical reasoning tasks. For instance, MathOctopus-7B improves its counterparts that trained on English from 42.2% to 50.8% on GSM8K testset. Codes are available at https://github.com/microsoft/MathOctopus.}, Year          = {2023}, Month         = {10}, Url           = {http://arxiv.org/pdf/2310.20246v5}, File          = {2310.20246v5.pdf} }
 
@article{2311.05113v1, Author        = {Haoyi Wu and Wenyang Hui and Yezeng Chen and Weiqi Wu and Kewei Tu and Yi Zhou}, Title         = {Conic10K: A Challenging Math Problem Understanding and Reasoning Dataset}, Eprint        = {http://arxiv.org/abs/2311.05113v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Mathematical understanding and reasoning are crucial tasks for assessing the capabilities of artificial intelligence (AI). However, existing benchmarks either require just a few steps of reasoning, or only contain a small amount of data in one specific topic, making it hard to analyse AI's behaviour with reference to different problems within a specific topic in detail. In this work, we propose Conic10K, a challenging math problem dataset on conic sections in Chinese senior high school education. Our dataset contains various problems with different reasoning depths, while only the knowledge from conic sections is required. Since the dataset only involves a narrow range of knowledge, it is easy to separately analyse the knowledge a model possesses and the reasoning ability it has. For each problem, we provide a high-quality formal representation, the reasoning steps, and the final solution. Experiments show that existing large language models, including GPT-4, exhibit weak performance on complex reasoning. We hope that our findings could inspire more advanced techniques for precise natural language understanding and reasoning. Our dataset and codes are available at https://github.com/whyNLP/Conic10K.}, Year          = {2023}, Month         = {11}, Url           = {http://arxiv.org/pdf/2311.05113v1}, File          = {2311.05113v1.pdf} }
 
@article{2310.11441v2, Author        = {Jianwei Yang and Hao Zhang and Feng Li and Xueyan Zou and Chunyuan Li and Jianfeng Gao}, Title         = {Set-of-Mark Prompting Unleashes Extraordinary Visual Grounding in GPT-4V}, Eprint        = {http://arxiv.org/abs/2310.11441v2}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CV}, Abstract      = {We present Set-of-Mark (SoM), a new visual prompting method, to unleash the visual grounding abilities of large multimodal models (LMMs), such as GPT-4V. As illustrated in Fig. 1 (right), we employ off-the-shelf interactive segmentation models, such as SEEM/SAM, to partition an image into regions at different levels of granularity, and overlay these regions with a set of marks e.g., alphanumerics, masks, boxes. Using the marked image as input, GPT-4V can answer the questions that require visual grounding. We perform a comprehensive empirical study to validate the effectiveness of SoM on a wide range of fine-grained vision and multimodal tasks. For example, our experiments show that GPT-4V with SoM in zero-shot setting outperforms the state-of-the-art fully-finetuned referring expression comprehension and segmentation model on RefCOCOg. Code for SoM prompting is made public at: https://github.com/microsoft/SoM.}, Year          = {2023}, Month         = {10}, Url           = {http://arxiv.org/pdf/2310.11441v2}, File          = {2310.11441v2.pdf} }
 
@article{2312.00784v2, Author        = {Mu Cai and Haotian Liu and Dennis Park and Siva Karthik Mustikovela and Gregory P. Meyer and Yuning Chai and Yong Jae Lee}, Title         = {ViP-LLaVA: Making Large Multimodal Models Understand Arbitrary Visual Prompts}, Eprint        = {http://arxiv.org/abs/2312.00784v2}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CV}, Abstract      = {While existing large vision-language multimodal models focus on whole image understanding, there is a prominent gap in achieving region-specific comprehension. Current approaches that use textual coordinates or spatial encodings often fail to provide a user-friendly interface for visual prompting. To address this challenge, we introduce a novel multimodal model capable of decoding arbitrary visual prompts. This allows users to intuitively mark images and interact with the model using natural cues like a "red bounding box"" or ""pointed arrow"". Our simple design directly overlays visual markers onto the RGB image, eliminating the need for complex region encodings, yet achieves state-of-the-art performance on region-understanding tasks like Visual7W, PointQA, and Visual Commonsense Reasoning benchmark. Furthermore, we present ViP-Bench, a comprehensive benchmark to assess the capability of models in understanding visual prompts across multiple dimensions, enabling future research in this domain. Code, data, and model are publicly available.}, Year          = {2023}, Month         = {12}, Url           = {http://arxiv.org/pdf/2312.00784v2}, File          = {2312.00784v2.pdf} }"
 
@article{2306.02858v4, Author        = {Hang Zhang and Xin Li and Lidong Bing}, Title         = {Video-LLaMA: An Instruction-tuned Audio-Visual Language Model for Video Understanding}, Eprint        = {http://arxiv.org/abs/2306.02858v4}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {We present Video-LLaMA a multi-modal framework that empowers Large Language Models (LLMs) with the capability of understanding both visual and auditory content in the video. Video-LLaMA bootstraps cross-modal training from the frozen pre-trained visual and audio encoders and the frozen LLMs. Unlike previous works that complement LLMs to process the visual or audio signals only, Video-LLaMA enables video comprehension by tackling two challenges: (1) capturing the temporal changes in visual scenes, (2) integrating audio-visual signals. To counter the first challenge, we propose a Video Q-former to assemble a pre-trained image encoder into our video encoder and introduce a video-to-text generation task to learn video-language correspondence. For the second challenge, we leverage ImageBind, a universal embedding model aligning multiple modalities, as the pre-trained audio encoder and introduce an Audio Q-former on top of ImageBind to learn reasonable auditory query embeddings for the LLM module. To align the output of both visual and audio encoders with LLM's embedding space, we first train Video-LLaMA on massive video/image-caption pairs and then tune our model with visual-instruction datasets of moderate amount but higher quality. We found Video-LLaMA shows the ability to perceive and comprehend video content and generate meaningful responses grounded in the visual and auditory information presented in the videos.}, Year          = {2023}, Month         = {6}, Url           = {http://arxiv.org/pdf/2306.02858v4}, File          = {2306.02858v4.pdf} }
 
@article{2311.10122v3, Author        = {Bin Lin and Yang Ye and Bin Zhu and Jiaxi Cui and Munan Ning and Peng Jin and Li Yuan}, Title         = {Video-LLaVA: Learning United Visual Representation by Alignment Before Projection}, Eprint        = {http://arxiv.org/abs/2311.10122v3}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CV}, Abstract      = {The Large Vision-Language Model (LVLM) has enhanced the performance of various downstream tasks in visual-language understanding. Most existing approaches encode images and videos into separate feature spaces, which are then fed as inputs to large language models. However, due to the lack of unified tokenization for images and videos, namely misalignment before projection, it becomes challenging for a Large Language Model (LLM) to learn multi-modal interactions from several poor projection layers. In this work, we unify visual representation into the language feature space to advance the foundational LLM towards a unified LVLM. As a result, we establish a simple but robust LVLM baseline, Video-LLaVA, which learns from a mixed dataset of images and videos, mutually enhancing each other. Video-LLaVA achieves superior performances on a broad range of 9 image benchmarks across 5 image question-answering datasets and 4 image benchmark toolkits. Additionally, our Video-LLaVA also outperforms Video-ChatGPT by 5.8%, 9.9%, 18.6%, and 10.1% on MSRVTT, MSVD, TGIF, and ActivityNet, respectively. Notably, extensive experiments demonstrate that Video-LLaVA mutually benefits images and videos within a unified visual representation, outperforming models designed specifically for images or videos. We aim for this work to provide modest insights into the multi-modal inputs for the LLM. Code address: \href{https://github.com/PKU-YuanGroup/Video-LLaVA}}, Year          = {2023}, Month         = {11}, Url           = {http://arxiv.org/pdf/2311.10122v3}, File          = {2311.10122v3.pdf} }
 
@article{2405.03770v1, Author        = {Neelu Madan and Andreas Moegelmose and Rajat Modi and Yogesh S. Rawat and Thomas B. Moeslund}, Title         = {Foundation Models for Video Understanding: A Survey}, Eprint        = {http://arxiv.org/abs/2405.03770v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CV}, Abstract      = {Video Foundation Models (ViFMs) aim to learn a general-purpose representation for various video understanding tasks. Leveraging large-scale datasets and powerful models, ViFMs achieve this by capturing robust and generic features from video data. This survey analyzes over 200 video foundational models, offering a comprehensive overview of benchmarks and evaluation metrics across 14 distinct video tasks categorized into 3 main categories. Additionally, we offer an in-depth performance analysis of these models for the 6 most common video tasks. We categorize ViFMs into three categories: 1) Image-based ViFMs, which adapt existing image models for video tasks, 2) Video-Based ViFMs, which utilize video-specific encoding methods, and 3) Universal Foundational Models (UFMs), which combine multiple modalities (image, video, audio, and text etc.) within a single framework. By comparing the performance of various ViFMs on different tasks, this survey offers valuable insights into their strengths and weaknesses, guiding future advancements in video understanding. Our analysis surprisingly reveals that image-based foundation models consistently outperform video-based models on most video understanding tasks. Additionally, UFMs, which leverage diverse modalities, demonstrate superior performance on video tasks. We share the comprehensive list of ViFMs studied in this work at: \url{https://github.com/NeeluMadan/ViFM_Survey.git}}, Year          = {2024}, Month         = {5}, Url           = {http://arxiv.org/pdf/2405.03770v1}, File          = {2405.03770v1.pdf} }
 
@article{2308.09583v2, Author        = {Haipeng Luo and Qingfeng Sun and Can Xu and Pu Zhao and Jianguang Lou and Chongyang Tao and Xiubo Geng and Qingwei Lin and Shifeng Chen and Yansong Tang and Dongmei Zhang}, Title         = {WizardMath: Empowering Mathematical Reasoning for Large Language Models via Reinforced Evol-Instruct}, Eprint        = {http://arxiv.org/abs/2308.09583v2}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Large language models (LLMs), such as GPT-4, have shown remarkable performance in natural language processing (NLP) tasks, including challenging mathematical reasoning. However, most existing open-source models are only pre-trained on large-scale internet data and without math-related optimization. In this paper, we present WizardMath, which enhances the mathematical CoT reasoning abilities of LLMs without using external python tools, by applying our proposed Reinforcement Learning from Evol-Instruct Feedback (RLEIF) method to the domain of math. Through extensive experiments on two mathematical reasoning benchmarks, namely GSM8k and MATH, we reveal the extraordinary capabilities of our model. Remarkably, WizardMath-Mistral 7B surpasses top-tier open-source LLMs by a substantial margin with higher data efficiency. Furthermore, WizardMath 70B even outperforms GPT-3.5-Turbo, Claude 2, Gemini Pro and GPT-4-early-version. Additionally, our preliminary exploration highlights the pivotal role of instruction evolution and process supervision in achieving exceptional math performance. For more details refer to https://github.com/nlpxucan/WizardLM}, Year          = {2023}, Month         = {8}, Url           = {http://arxiv.org/pdf/2308.09583v2}, File          = {2308.09583v2.pdf} }
 
@article{2306.05424v2, Author        = {Muhammad Maaz and Hanoona Rasheed and Salman Khan and Fahad Shahbaz Khan}, Title         = {Video-ChatGPT: Towards Detailed Video Understanding via Large Vision and Language Models}, Eprint        = {http://arxiv.org/abs/2306.05424v2}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CV}, Abstract      = {Conversation agents fueled by Large Language Models (LLMs) are providing a new way to interact with visual data. While there have been initial attempts for image-based conversation models, this work addresses the under-explored field of \emph{video-based conversation} by introducing Video-ChatGPT. It is a multimodal model that merges a video-adapted visual encoder with an LLM. The resulting model is capable of understanding and generating detailed conversations about videos. We introduce a new dataset of 100,000 video-instruction pairs used to train Video-ChatGPT acquired via manual and semi-automated pipeline that is easily scalable and robust to label noise. We also develop a quantitative evaluation framework for video-based dialogue models to objectively analyze the strengths and weaknesses of video-based dialogue models. Code: https://github.com/mbzuai-oryx/Video-ChatGPT.}, Year          = {2023}, Month         = {6}, Url           = {http://arxiv.org/pdf/2306.05424v2}, File          = {2306.05424v2.pdf} }
 
@article{2305.15334v1, Author        = {Shishir G. Patil and Tianjun Zhang and Xin Wang and Joseph E. Gonzalez}, Title         = {Gorilla: Large Language Model Connected with Massive APIs}, Eprint        = {http://arxiv.org/abs/2305.15334v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Large Language Models (LLMs) have seen an impressive wave of advances recently, with models now excelling in a variety of tasks, such as mathematical reasoning and program synthesis. However, their potential to effectively use tools via API calls remains unfulfilled. This is a challenging task even for today's state-of-the-art LLMs such as GPT-4, largely due to their inability to generate accurate input arguments and their tendency to hallucinate the wrong usage of an API call. We release Gorilla, a finetuned LLaMA-based model that surpasses the performance of GPT-4 on writing API calls. When combined with a document retriever, Gorilla demonstrates a strong capability to adapt to test-time document changes, enabling flexible user updates or version changes. It also substantially mitigates the issue of hallucination, commonly encountered when prompting LLMs directly. To evaluate the model's ability, we introduce APIBench, a comprehensive dataset consisting of HuggingFace, TorchHub, and TensorHub APIs. The successful integration of the retrieval system with Gorilla demonstrates the potential for LLMs to use tools more accurately, keep up with frequently updated documentation, and consequently increase the reliability and applicability of their outputs. Gorilla's code, model, data, and demo are available at https://gorilla.cs.berkeley.edu}, Year          = {2023}, Month         = {5}, Url           = {http://arxiv.org/pdf/2305.15334v1}, File          = {2305.15334v1.pdf} }
 
@article{2304.14178v3, Author        = {Qinghao Ye and Haiyang Xu and Guohai Xu and Jiabo Ye and Ming Yan and Yiyang Zhou and Junyang Wang and Anwen Hu and Pengcheng Shi and Yaya Shi and Chenliang Li and Yuanhong Xu and Hehong Chen and Junfeng Tian and Qi Qian and Ji Zhang and Fei Huang and Jingren Zhou}, Title         = {mPLUG-Owl: Modularization Empowers Large Language Models with Multimodality}, Eprint        = {http://arxiv.org/abs/2304.14178v3}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Large language models (LLMs) have demonstrated impressive zero-shot abilities on a variety of open-ended tasks, while recent research has also explored the use of LLMs for multi-modal generation. In this study, we introduce mPLUG-Owl, a novel training paradigm that equips LLMs with multi-modal abilities through modularized learning of foundation LLM, a visual knowledge module, and a visual abstractor module. This approach can support multiple modalities and facilitate diverse unimodal and multimodal abilities through modality collaboration. The training paradigm of mPLUG-Owl involves a two-stage method for aligning image and text, which learns visual knowledge with the assistance of LLM while maintaining and even improving the generation abilities of LLM. In the first stage, the visual knowledge module and abstractor module are trained with a frozen LLM module to align the image and text. In the second stage, language-only and multi-modal supervised datasets are used to jointly fine-tune a low-rank adaption (LoRA) module on LLM and the abstractor module by freezing the visual knowledge module. We carefully build a visually-related instruction evaluation set OwlEval. Experimental results show that our model outperforms existing multi-modal models, demonstrating mPLUG-Owl's impressive instruction and visual understanding ability, multi-turn conversation ability, and knowledge reasoning ability. Besides, we observe some unexpected and exciting abilities such as multi-image correlation and scene text understanding, which makes it possible to leverage it for harder real scenarios, such as vision-only document comprehension. Our code, pre-trained model, instruction-tuned models, and evaluation set are available at https://github.com/X-PLUG/mPLUG-Owl. The online demo is available at https://www.modelscope.cn/studios/damo/mPLUG-Owl.}, Year          = {2023}, Month         = {4}, Url           = {http://arxiv.org/pdf/2304.14178v3}, File          = {2304.14178v3.pdf} }
 
@article{2009.03300v3, Author        = {Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt}, Title         = {Measuring Massive Multitask Language Understanding}, Eprint        = {http://arxiv.org/abs/2009.03300v3}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CY}, Abstract      = {We propose a new test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more. To attain high accuracy on this test, models must possess extensive world knowledge and problem solving ability. We find that while most recent models have near random-chance accuracy, the very largest GPT-3 model improves over random chance by almost 20 percentage points on average. However, on every one of the 57 tasks, the best models still need substantial improvements before they can reach expert-level accuracy. Models also have lopsided performance and frequently do not know when they are wrong. Worse, they still have near-random accuracy on some socially important subjects such as morality and law. By comprehensively evaluating the breadth and depth of a model's academic and professional understanding, our test can be used to analyze models across many tasks and to identify important shortcomings.}, Year          = {2020}, Month         = {9}, Url           = {http://arxiv.org/pdf/2009.03300v3}, File          = {2009.03300v3.pdf} }
 
@article{2406.01574v6, Author        = {Yubo Wang and Xueguang Ma and Ge Zhang and Yuansheng Ni and Abhranil Chandra and Shiguang Guo and Weiming Ren and Aaran Arulraj and Xuan He and Ziyan Jiang and Tianle Li and Max Ku and Kai Wang and Alex Zhuang and Rongqi Fan and Xiang Yue and Wenhu Chen}, Title         = {MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark}, Eprint        = {http://arxiv.org/abs/2406.01574v6}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {In the age of large-scale language models, benchmarks like the Massive Multitask Language Understanding (MMLU) have been pivotal in pushing the boundaries of what AI can achieve in language comprehension and reasoning across diverse domains. However, as models continue to improve, their performance on these benchmarks has begun to plateau, making it increasingly difficult to discern differences in model capabilities. This paper introduces MMLU-Pro, an enhanced dataset designed to extend the mostly knowledge-driven MMLU benchmark by integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. Additionally, MMLU-Pro eliminates the trivial and noisy questions in MMLU. Our experimental results show that MMLU-Pro not only raises the challenge, causing a significant drop in accuracy by 16% to 33% compared to MMLU but also demonstrates greater stability under varying prompts. With 24 different prompt styles tested, the sensitivity of model scores to prompt variations decreased from 4-5% in MMLU to just 2% in MMLU-Pro. Additionally, we found that models utilizing Chain of Thought (CoT) reasoning achieved better performance on MMLU-Pro compared to direct answering, which is in stark contrast to the findings on the original MMLU, indicating that MMLU-Pro includes more complex reasoning questions. Our assessments confirm that MMLU-Pro is a more discriminative benchmark to better track progress in the field.}, Year          = {2024}, Month         = {6}, Url           = {http://arxiv.org/pdf/2406.01574v6}, File          = {2406.01574v6.pdf} }
 
@article{2402.01781v2, Author        = {Norah Alzahrani and Hisham Abdullah Alyahya and Yazeed Alnumay and Sultan Alrashed and Shaykhah Alsubaie and Yusef Almushaykeh and Faisal Mirza and Nouf Alotaibi and Nora Altwairesh and Areeb Alowisheq and M Saiful Bari and Haidar Khan}, Title         = {When Benchmarks are Targets: Revealing the Sensitivity of Large Language Model Leaderboards}, Eprint        = {http://arxiv.org/abs/2402.01781v2}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Large Language Model (LLM) leaderboards based on benchmark rankings are regularly used to guide practitioners in model selection. Often, the published leaderboard rankings are taken at face value - we show this is a (potentially costly) mistake. Under existing leaderboards, the relative performance of LLMs is highly sensitive to (often minute) details. We show that for popular multiple-choice question benchmarks (e.g., MMLU), minor perturbations to the benchmark, such as changing the order of choices or the method of answer selection, result in changes in rankings up to 8 positions. We explain this phenomenon by conducting systematic experiments over three broad categories of benchmark perturbations and identifying the sources of this behavior. Our analysis results in several best-practice recommendations, including the advantage of a hybrid scoring method for answer selection. Our study highlights the dangers of relying on simple benchmark evaluations and charts the path for more robust evaluation schemes on the existing benchmarks. The code for this paper is available at https://github.com/National-Center-for-AI-Saudi-Arabia/lm-evaluation-harness.}, Year          = {2024}, Month         = {2}, Url           = {http://arxiv.org/pdf/2402.01781v2}, File          = {2402.01781v2.pdf} }
 
@article{2307.11088v3, Author        = {Chenxin An and Shansan Gong and Ming Zhong and Xingjian Zhao and Mukai Li and Jun Zhang and Lingpeng Kong and Xipeng Qiu}, Title         = {L-Eval: Instituting Standardized Evaluation for Long Context Language Models}, Eprint        = {http://arxiv.org/abs/2307.11088v3}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Recently, there has been growing interest in extending the context length of large language models (LLMs), aiming to effectively process long inputs of one turn or conversations with more extensive histories. While proprietary models such as GPT-4 and Claude can largely preserve the reasoning ability in an extended context, open-source models are still progressing through the early stages of development. To bridge this gap, we propose L-Eval to institute a more standardized evaluation for long context language models (LCLMs) addressing two key aspects: dataset construction and evaluation metrics. On the one hand, we build a new evaluation suite containing 20 sub-tasks, 508 long documents, and over 2,000 human-labeled query-response pairs encompassing diverse question styles, domains, and input length (3k$\sim$200k tokens). On the other hand, we investigate the effectiveness in evalution metrics for LCLMs. Results show that popular n-gram matching metrics generally can not correlate well with human judgment, and thus we strongly advocate for length-instruction-enhanced (LIE) evaluation and employing LLM judges. We conducted a comprehensive study of 4 popular commercial LLMs and 12 open-source counterparts using the L-Eval benchmark. Our empirical findings offer useful insights into the study of LCLMs and lay the groundwork for the development of more principled evaluation of these models.}, Year          = {2023}, Month         = {7}, Url           = {http://arxiv.org/pdf/2307.11088v3}, File          = {2307.11088v3.pdf} }
 
@article{2311.12983v1, Author        = {Grégoire Mialon and Clémentine Fourrier and Craig Swift and Thomas Wolf and Yann LeCun and Thomas Scialom}, Title         = {GAIA: a benchmark for General AI Assistants}, Eprint        = {http://arxiv.org/abs/2311.12983v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {We introduce GAIA, a benchmark for General AI Assistants that, if solved, would represent a milestone in AI research. GAIA proposes real-world questions that require a set of fundamental abilities such as reasoning, multi-modality handling, web browsing, and generally tool-use proficiency. GAIA questions are conceptually simple for humans yet challenging for most advanced AIs: we show that human respondents obtain 92\% vs. 15\% for GPT-4 equipped with plugins. This notable performance disparity contrasts with the recent trend of LLMs outperforming humans on tasks requiring professional skills in e.g. law or chemistry. GAIA's philosophy departs from the current trend in AI benchmarks suggesting to target tasks that are ever more difficult for humans. We posit that the advent of Artificial General Intelligence (AGI) hinges on a system's capability to exhibit similar robustness as the average human does on such questions. Using GAIA's methodology, we devise 466 questions and their answer. We release our questions while retaining answers to 300 of them to power a leader-board available at https://huggingface.co/gaia-benchmark.}, Year          = {2023}, Month         = {11}, Url           = {http://arxiv.org/pdf/2311.12983v1}, File          = {2311.12983v1.pdf} }
 
@article{2312.04724v1, Author        = {Manish Bhatt and Sahana Chennabasappa and Cyrus Nikolaidis and Shengye Wan and Ivan Evtimov and Dominik Gabi and Daniel Song and Faizan Ahmad and Cornelius Aschermann and Lorenzo Fontana and Sasha Frolov and Ravi Prakash Giri and Dhaval Kapil and Yiannis Kozyrakis and David LeBlanc and James Milazzo and Aleksandar Straumann and Gabriel Synnaeve and Varun Vontimitta and Spencer Whitman and Joshua Saxe}, Title         = {Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models}, Eprint        = {http://arxiv.org/abs/2312.04724v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CR}, Abstract      = {This paper presents CyberSecEval, a comprehensive benchmark developed to help bolster the cybersecurity of Large Language Models (LLMs) employed as coding assistants. As what we believe to be the most extensive unified cybersecurity safety benchmark to date, CyberSecEval provides a thorough evaluation of LLMs in two crucial security domains: their propensity to generate insecure code and their level of compliance when asked to assist in cyberattacks. Through a case study involving seven models from the Llama 2, Code Llama, and OpenAI GPT large language model families, CyberSecEval effectively pinpointed key cybersecurity risks. More importantly, it offered practical insights for refining these models. A significant observation from the study was the tendency of more advanced models to suggest insecure code, highlighting the critical need for integrating security considerations in the development of sophisticated LLMs. CyberSecEval, with its automated test case generation and evaluation pipeline covers a broad scope and equips LLM designers and researchers with a tool to broadly measure and enhance the cybersecurity safety properties of LLMs, contributing to the development of more secure AI systems.}, Year          = {2023}, Month         = {12}, Url           = {http://arxiv.org/pdf/2312.04724v1}, File          = {2312.04724v1.pdf} }
 
@article{2406.10229v1, Author        = {Lovish Madaan and Aaditya K. Singh and Rylan Schaeffer and Andrew Poulton and Sanmi Koyejo and Pontus Stenetorp and Sharan Narang and Dieuwke Hupkes}, Title         = {Quantifying Variance in Evaluation Benchmarks}, Eprint        = {http://arxiv.org/abs/2406.10229v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.LG}, Abstract      = {Evaluation benchmarks are the cornerstone of measuring capabilities of large language models (LLMs), as well as driving progress in said capabilities. Originally designed to make claims about capabilities (or lack thereof) in fully pretrained models, evaluation benchmarks are now also extensively used to decide between various training choices. Despite this widespread usage, we rarely quantify the variance in our evaluation benchmarks, which dictates whether differences in performance are meaningful. Here, we define and measure a range of metrics geared towards measuring variance in evaluation benchmarks, including seed variance across initialisations, and monotonicity during training. By studying a large number of models -- both openly available and pretrained from scratch -- we provide empirical estimates for a variety of variance metrics, with considerations and recommendations for practitioners. We also evaluate the utility and tradeoffs of continuous versus discrete performance measures and explore options for better understanding and reducing this variance. We find that simple changes, such as framing choice tasks (like MMLU) as completion tasks, can often reduce variance for smaller scale ($\sim$7B) models, while more involved methods inspired from human testing literature (such as item analysis and item response theory) struggle to meaningfully reduce variance. Overall, our work provides insights into variance in evaluation benchmarks, suggests LM-specific techniques to reduce variance, and more generally encourages practitioners to carefully factor in variance when comparing models.}, Year          = {2024}, Month         = {6}, Url           = {http://arxiv.org/pdf/2406.10229v1}, File          = {2406.10229v1.pdf} }
 
@article{2310.13486v1, Author        = {Lucas Weber and Elia Bruni and Dieuwke Hupkes}, Title         = {Mind the instructions: a holistic evaluation of consistency and interactions in prompt-based learning}, Eprint        = {http://arxiv.org/abs/2310.13486v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Finding the best way of adapting pre-trained language models to a task is a big challenge in current NLP. Just like the previous generation of task-tuned models (TT), models that are adapted to tasks via in-context-learning (ICL) are robust in some setups but not in others. Here, we present a detailed analysis of which design choices cause instabilities and inconsistencies in LLM predictions. First, we show how spurious correlations between input distributions and labels -- a known issue in TT models -- form only a minor problem for prompted models. Then, we engage in a systematic, holistic evaluation of different factors that have been found to influence predictions in a prompting setup. We test all possible combinations of a range of factors on both vanilla and instruction-tuned (IT) LLMs of different scale and statistically analyse the results to show which factors are the most influential, interactive or stable. Our results show which factors can be used without precautions and which should be avoided or handled with care in most settings.}, Year          = {2023}, Month         = {10}, Url           = {http://arxiv.org/pdf/2310.13486v1}, File          = {2310.13486v1.pdf} }
 
@article{2309.03882v4, Author        = {Chujie Zheng and Hao Zhou and Fandong Meng and Jie Zhou and Minlie Huang}, Title         = {Large Language Models Are Not Robust Multiple Choice Selectors}, Eprint        = {http://arxiv.org/abs/2309.03882v4}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {Multiple choice questions (MCQs) serve as a common yet important task format in the evaluation of large language models (LLMs). This work shows that modern LLMs are vulnerable to option position changes in MCQs due to their inherent "selection bias"", namely, they prefer to select specific option IDs as answers (like ""Option A""). Through extensive empirical analyses with 20 LLMs on three benchmarks, we pinpoint that this behavioral bias primarily stems from LLMs' token bias, where the model a priori assigns more probabilistic mass to specific option ID tokens (e.g., A/B/C/D) when predicting answers from the option IDs. To mitigate selection bias, we propose a label-free, inference-time debiasing method, called PriDe, which separates the model's prior bias for option IDs from the overall prediction distribution. PriDe first estimates the prior by permutating option contents on a small number of test samples, and then applies the estimated prior to debias the remaining samples. We demonstrate that it achieves interpretable and transferable debiasing with high computational efficiency. We hope this work can draw broader research attention to the bias and robustness of modern LLMs.}, Year          = {2023}, Month         = {9}, Url           = {http://arxiv.org/pdf/2309.03882v4}, File          = {2309.03882v4.pdf} }"
 
@article{2402.16822v3, Author        = {Mikayel Samvelyan and Sharath Chandra Raparthy and Andrei Lupu and Eric Hambro and Aram H. Markosyan and Manish Bhatt and Yuning Mao and Minqi Jiang and Jack Parker-Holder and Jakob Foerster and Tim Rocktäschel and Roberta Raileanu}, Title         = {Rainbow Teaming: Open-Ended Generation of Diverse Adversarial Prompts}, Eprint        = {http://arxiv.org/abs/2402.16822v3}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CL}, Abstract      = {As large language models (LLMs) become increasingly prevalent across many real-world applications, understanding and enhancing their robustness to adversarial attacks is of paramount importance. Existing methods for identifying adversarial prompts tend to focus on specific domains, lack diversity, or require extensive human annotations. To address these limitations, we present Rainbow Teaming, a novel black-box approach for producing a diverse collection of adversarial prompts. Rainbow Teaming casts adversarial prompt generation as a quality-diversity problem and uses open-ended search to generate prompts that are both effective and diverse. Focusing on the safety domain, we use Rainbow Teaming to target various state-of-the-art LLMs, including the Llama 2 and Llama 3 models. Our approach reveals hundreds of effective adversarial prompts, with an attack success rate exceeding 90% across all tested models. Furthermore, we demonstrate that prompts generated by Rainbow Teaming are highly transferable and that fine-tuning models with synthetic data generated by our method significantly enhances their safety without sacrificing general performance or helpfulness. We additionally explore the versatility of Rainbow Teaming by applying it to question answering and cybersecurity, showcasing its potential to drive robust open-ended self-improvement in a wide range of applications.}, Year          = {2024}, Month         = {2}, Url           = {http://arxiv.org/pdf/2402.16822v3}, File          = {2402.16822v3.pdf} }
 
@article{2310.08419v4, Author        = {Patrick Chao and Alexander Robey and Edgar Dobriban and Hamed Hassani and George J. Pappas and Eric Wong}, Title         = {Jailbreaking Black Box Large Language Models in Twenty Queries}, Eprint        = {http://arxiv.org/abs/2310.08419v4}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.LG}, Abstract      = {There is growing interest in ensuring that large language models (LLMs) align with human values. However, the alignment of such models is vulnerable to adversarial jailbreaks, which coax LLMs into overriding their safety guardrails. The identification of these vulnerabilities is therefore instrumental in understanding inherent weaknesses and preventing future misuse. To this end, we propose Prompt Automatic Iterative Refinement (PAIR), an algorithm that generates semantic jailbreaks with only black-box access to an LLM. PAIR -- which is inspired by social engineering attacks -- uses an attacker LLM to automatically generate jailbreaks for a separate targeted LLM without human intervention. In this way, the attacker LLM iteratively queries the target LLM to update and refine a candidate jailbreak. Empirically, PAIR often requires fewer than twenty queries to produce a jailbreak, which is orders of magnitude more efficient than existing algorithms. PAIR also achieves competitive jailbreaking success rates and transferability on open and closed-source LLMs, including GPT-3.5/4, Vicuna, and Gemini.}, Year          = {2023}, Month         = {10}, Url           = {http://arxiv.org/pdf/2310.08419v4}, File          = {2310.08419v4.pdf} }
 
@article{2404.13208v1, Author        = {Eric Wallace and Kai Xiao and Reimar Leike and Lilian Weng and Johannes Heidecke and Alex Beutel}, Title         = {The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions}, Eprint        = {http://arxiv.org/abs/2404.13208v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.CR}, Abstract      = {Today's LLMs are susceptible to prompt injections, jailbreaks, and other attacks that allow adversaries to overwrite a model's original instructions with their own malicious prompts. In this work, we argue that one of the primary vulnerabilities underlying these attacks is that LLMs often consider system prompts (e.g., text from an application developer) to be the same priority as text from untrusted users and third parties. To address this, we propose an instruction hierarchy that explicitly defines how models should behave when instructions of different priorities conflict. We then propose a data generation method to demonstrate this hierarchical instruction following behavior, which teaches LLMs to selectively ignore lower-privileged instructions. We apply this method to GPT-3.5, showing that it drastically increases robustness -- even for attack types not seen during training -- while imposing minimal degradations on standard capabilities.}, Year          = {2024}, Month         = {4}, Url           = {http://arxiv.org/pdf/2404.13208v1}, File          = {2404.13208v1.pdf} }
 
@article{2311.17035v1, Author        = {Milad Nasr and Nicholas Carlini and Jonathan Hayase and Matthew Jagielski and A. Feder Cooper and Daphne Ippolito and Christopher A. Choquette-Choo and Eric Wallace and Florian Tramèr and Katherine Lee}, Title         = {Scalable Extraction of Training Data from (Production) Language Models}, Eprint        = {http://arxiv.org/abs/2311.17035v1}, ArchivePrefix = {arXiv}, PrimaryClass  = {cs.LG}, Abstract      = {This paper studies extractable memorization: training data that an adversary can efficiently extract by querying a machine learning model without prior knowledge of the training dataset. We show an adversary can extract gigabytes of training data from open-source language models like Pythia or GPT-Neo, semi-open models like LLaMA or Falcon, and closed models like ChatGPT. Existing techniques from the literature suffice to attack unaligned models; in order to attack the aligned ChatGPT, we develop a new divergence attack that causes the model to diverge from its chatbot-style generations and emit training data at a rate 150x higher than when behaving properly. Our methods show practical attacks can recover far more data than previously thought, and reveal that current alignment techniques do not eliminate memorization.}, Year          = {2023}, Month         = {11}, Url           = {http://arxiv.org/pdf/2311.17035v1}, File          = {2311.17035v1.pdf} }