PyTerrier demonstration for msmarco_passage

In [ ]:
import pyterrier as pt
if not pt.started():
    pt.init()

systems=[]
names=[]
dataset = pt.get_dataset('msmarco_passage')

Systems using index variant terrier_stemmed

In [ ]:
bm25_terrier_stemmed = pt.BatchRetrieve.from_dataset('msmarco_passage', 'terrier_stemmed', wmodel='BM25')
systems.append(bm25_terrier_stemmed)
names.append('bm25_terrier_stemmed')


dph_terrier_stemmed = pt.BatchRetrieve.from_dataset('msmarco_passage', 'terrier_stemmed', wmodel='DPH')
systems.append(dph_terrier_stemmed)
names.append('dph_terrier_stemmed')

Systems using index variant terrier_stemmed_docT5query

In [ ]:
bm25_terrier_stemmed_docT5query = pt.BatchRetrieve.from_dataset('msmarco_passage', 'terrier_stemmed_docT5query', wmodel='BM25')
systems.append(bm25_terrier_stemmed_docT5query)
names.append('bm25_terrier_stemmed_docT5query')

Systems using index variant terrier_stemmed_deepct

In [ ]:
bm25_terrier_stemmed_deepct = pt.BatchRetrieve.from_dataset('msmarco_passage', 'terrier_stemmed_deepct', wmodel='BM25')
systems.append(bm25_terrier_stemmed_deepct)
names.append('bm25_terrier_stemmed_deepct')

Evaluation on trec-2019 topics and qrels

43 topics used in the TREC Deep Learning track Passage Ranking task, with deep judgements

In [ ]:
pt.Experiment(
    systems,
    pt.get_dataset('msmarco_passage').get_topics('test-2019'),
    pt.get_dataset('msmarco_passage').get_qrels('test-2019'),
    batch_size=200,
    drop_unused=True,
    eval_metrics=['ndcg_cut_10'],
    names=names)

Evaluation on trec-2020 topics and qrels

43 topics used in the TREC Deep Learning track Passage Ranking task, with deep judgements

In [ ]:
pt.Experiment(
    systems,
    pt.get_dataset('trec-deep-learning-passages').get_topics('test-2020'),
    pt.get_dataset('trec-deep-learning-passages').get_qrels('test-2020'),
    batch_size=200,
    drop_unused=True,
    eval_metrics=['ndcg_cut_10'],
    names=names)

Evaluation on dev.small topics and qrels

6800 topics with sparse judgements

In [ ]:
pt.Experiment(
    systems,
    pt.get_dataset('msmarco_passage').get_topics('dev.small'),
    pt.get_dataset('msmarco_passage').get_qrels('dev.small'),
    batch_size=200,
    drop_unused=True,
    eval_metrics=['recip_rank'],
    names=names)