This notebook demonstrates retrieval using PyTerrier on the MSMARCO v2 Passage Ranking corpus.
About the corpus: A revised corpus of 138M passages released by Microsoft in July 2021, which should be ranked based on their relevance to questions. Also used by the TREC 2021 Deep Learning track.
#!pip install -q python-terrier
import pyterrier as pt
if not pt.started():
pt.init()
from pyterrier.measures import *
dataset = pt.get_dataset('msmarcov2_passage')
Terrier's default Porter stemming, and stopwords removed.
#!pip install git+https://github.com/Georgetown-IR-Lab/OpenNIR.git
import onir_pt
# Lets use a Vanilla BERT ranker from OpenNIR. We'll use the Capreolus model available from Huggingface
vanilla_bert = onir_pt.reranker('hgf4_joint', ranker_config={'model': 'Capreolus/bert-base-msmarco', 'norm': 'softmax-2'})
bm25_terrier_stemmed = pt.BatchRetrieve.from_dataset('msmarcov2_passage', 'terrier_stemmed', wmodel='BM25')
bm25_bert_terrier_stemmed = (
pt.BatchRetrieve.from_dataset('msmarcov2_passage', 'terrier_stemmed')
>> pt.text.get_text(pt.get_dataset('irds:msmarco-passage-v2'), 'text')
>> onir_pt.reranker('hgf4_joint', ranker_config={'model': 'Capreolus/bert-base-msmarco', 'norm': 'softmax-2'}))
dph_terrier_stemmed = pt.BatchRetrieve.from_dataset('msmarcov2_passage', 'terrier_stemmed', wmodel='DPH')
dph_bert_terrier_stemmed = (
pt.BatchRetrieve.from_dataset('msmarcov2_passage', 'terrier_stemmed', wmodel='DPH')
>> pt.text.get_text(pt.get_dataset('irds:msmarco-passage-v2'), 'text')
>> onir_pt.reranker('hgf4_joint', ranker_config={'model': 'Capreolus/bert-base-msmarco', 'norm': 'softmax-2'}))
Terrier index, no stemming, no stopword removal.
#!pip install git+https://github.com/Georgetown-IR-Lab/OpenNIR.git
import onir_pt
# Lets use a Vanilla BERT ranker from OpenNIR. We'll use the Capreolus model available from Huggingface
vanilla_bert = onir_pt.reranker('hgf4_joint', ranker_config={'model': 'Capreolus/bert-base-msmarco', 'norm': 'softmax-2'})
bm25_terrier_unstemmed = pt.BatchRetrieve.from_dataset('msmarcov2_passage', 'terrier_unstemmed', wmodel='BM25')
bm25_bert_terrier_unstemmed = (
pt.BatchRetrieve.from_dataset('msmarcov2_passage', 'terrier_unstemmed')
>> pt.text.get_text(pt.get_dataset('irds:msmarco-passage-v2'), 'text')
>> onir_pt.reranker('hgf4_joint', ranker_config={'model': 'Capreolus/bert-base-msmarco', 'norm': 'softmax-2'}))
dph_terrier_unstemmed = pt.BatchRetrieve.from_dataset('msmarcov2_passage', 'terrier_unstemmed', wmodel='DPH')
dph_bert_terrier_unstemmed = (
pt.BatchRetrieve.from_dataset('msmarcov2_passage', 'terrier_unstemmed', wmodel='DPH')
>> pt.text.get_text(pt.get_dataset('irds:msmarco-passage-v2'), 'text')
>> onir_pt.reranker('hgf4_joint', ranker_config={'model': 'Capreolus/bert-base-msmarco', 'norm': 'softmax-2'}))
4,552 topics with sparse judgements
pt.Experiment(
[bm25_terrier_stemmed, bm25_bert_terrier_stemmed, dph_terrier_stemmed, dph_bert_terrier_stemmed, bm25_terrier_unstemmed, bm25_bert_terrier_unstemmed, dph_terrier_unstemmed, dph_bert_terrier_unstemmed],
pt.get_dataset('msmarcov2_passage').get_topics('dev1'),
pt.get_dataset('msmarcov2_passage').get_qrels('dev1'),
batch_size=200,
filter_by_qrels=True,
eval_metrics=[RR@10],
names=['bm25_terrier_stemmed', 'bm25_bert_terrier_stemmed', 'dph_terrier_stemmed', 'dph_bert_terrier_stemmed', 'bm25_terrier_unstemmed', 'bm25_bert_terrier_unstemmed', 'dph_terrier_unstemmed', 'dph_bert_terrier_unstemmed'])