2022
|
Mungmeeprued, Thisanaporn; Ma, Yuxin; Mehta, Nisarg; Lipani, Aldo Tab this Folder of Documents: Page Stream Segmentation of Business Documents Inproceedings Proceedings of the ACM Symposium on Document Engineering, ACM, San Jose, CA, USA, 2022. Abstract | Links | BibTeX @inproceedings{mungmeeprued-etal-2022-tab-this,
title = {Tab this Folder of Documents: Page Stream Segmentation of Business Documents},
author = {Thisanaporn Mungmeeprued and Yuxin Ma and Nisarg Mehta and Aldo Lipani},
url = {https://www.researchgate.net/publication/363113372_Tab_this_Folder_of_Documents_Page_Stream_Segmentation_of_Business_Documents},
year = {2022},
date = {2022-09-20},
urldate = {2022-09-20},
booktitle = {Proceedings of the ACM Symposium on Document Engineering},
publisher = {ACM},
address = {San Jose, CA, USA},
series = {DocEng '22},
abstract = {In the midst of digital transformation, automatically understanding the structure and composition of scanned documents is important in order to allow correct indexing, archiving, and processing. In many organizations, different types of documents are usually scanned together in folders, so it is essential to automate the task of segmenting the folders into documents which then proceed to further analysis tailored to specific document types. This task is known as Page Stream Segmentation (PSS). In this paper, we propose a deep learning solution to solve the task of determining whether or not a page is a breaking-point given a sequence of scanned pages (a folder) as input. We also provide a dataset called TABME (TAB this folder of docuMEnts) generated specifically for this task. Our proposed architecture combines LayoutLM and ResNet to exploit both textual and visual features of the document pages and achieves an F1 score of 0.953. The dataset and code used to run the experiments in this paper are available at the following web link: https://github.com/aldolipani/TABME.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
In the midst of digital transformation, automatically understanding the structure and composition of scanned documents is important in order to allow correct indexing, archiving, and processing. In many organizations, different types of documents are usually scanned together in folders, so it is essential to automate the task of segmenting the folders into documents which then proceed to further analysis tailored to specific document types. This task is known as Page Stream Segmentation (PSS). In this paper, we propose a deep learning solution to solve the task of determining whether or not a page is a breaking-point given a sequence of scanned pages (a folder) as input. We also provide a dataset called TABME (TAB this folder of docuMEnts) generated specifically for this task. Our proposed architecture combines LayoutLM and ResNet to exploit both textual and visual features of the document pages and achieves an F1 score of 0.953. The dataset and code used to run the experiments in this paper are available at the following web link: https://github.com/aldolipani/TABME. |
Kim, To Eun; Lipani, Aldo A Multi-Task Based Neural Model to Simulate Users in Goal-Oriented Dialogue Systems Inproceedings Proc.~of SIGIR, 2022. Abstract | Links | BibTeX @inproceedings{kim-lipani-2022-multi,
title = {A Multi-Task Based Neural Model to Simulate Users in Goal-Oriented Dialogue Systems},
author = {To Eun Kim and Aldo Lipani},
url = {https://www.researchgate.net/publication/360276605_A_Multi-Task_Based_Neural_Model_to_Simulate_Users_in_Goal-Oriented_Dialogue_Systems},
year = {2022},
date = {2022-06-11},
urldate = {2022-01-01},
booktitle = {Proc.~of SIGIR},
series = {SIGIR '22},
abstract = {A human-like user simulator that anticipates users' satisfaction scores, actions, and utterances can help goal-oriented dialogue systems in evaluating the conversation and refining their dialogue strategies. However, little work has experimented with user simulators which can generate users' utterances. In this paper, we propose a deep learning-based user simulator that predicts users' satisfaction scores and actions while also jointly generating users' utterances in a multi-task manner. In particular, we show that 1) the proposed deep text-to-text multi-task neural model achieves state-of-the-art performance in the users' satisfaction scores and actions prediction tasks, and 2) in an ablation analysis, user satisfaction score prediction, action prediction, and utterance generation tasks can boost the performance with each other via positive transfers across the tasks. The source code and model checkpoints used for the experiments run in this paper are available at the following weblink: https://github.com/kimdanny/user-simulation-t5.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
A human-like user simulator that anticipates users' satisfaction scores, actions, and utterances can help goal-oriented dialogue systems in evaluating the conversation and refining their dialogue strategies. However, little work has experimented with user simulators which can generate users' utterances. In this paper, we propose a deep learning-based user simulator that predicts users' satisfaction scores and actions while also jointly generating users' utterances in a multi-task manner. In particular, we show that 1) the proposed deep text-to-text multi-task neural model achieves state-of-the-art performance in the users' satisfaction scores and actions prediction tasks, and 2) in an ablation analysis, user satisfaction score prediction, action prediction, and utterance generation tasks can boost the performance with each other via positive transfers across the tasks. The source code and model checkpoints used for the experiments run in this paper are available at the following weblink: https://github.com/kimdanny/user-simulation-t5. |
Shi, Zhengxiang; Zhang, Qiang; Lipani, Aldo StepGame: A New Benchmark for Robust Multi-Hop Spatial Reasoning in Texts Inproceedings Proceedings of the Association for the Advancement of Artificial Intelligence, 2022. Abstract | Links | BibTeX @inproceedings{Shi2022,
title = {StepGame: A New Benchmark for Robust Multi-Hop Spatial Reasoning in Texts},
author = {Zhengxiang Shi and Qiang Zhang and Aldo Lipani},
url = {https://www.researchgate.net/publication/357159030_StepGame_A_New_Benchmark_for_Robust_Multi-Hop_Spatial_Reasoning_in_Texts},
year = {2022},
date = {2022-01-01},
booktitle = {Proceedings of the Association for the Advancement of Artificial Intelligence},
series = {AAAI '22},
abstract = {Inferring spatial relations in natural language is a crucial ability an intelligent system should possess. The bAbI dataset tries to capture tasks relevant to this domain (tasks 17 and 19). However, these tasks have several limitations. Most importantly, they are limited to fixed expressions, they are limited in the number of reasoning steps required to solve them, and they fail to test the robustness of models to input that contains irrelevant or redundant information. In this paper, we present a new Question-Answering dataset called StepGame for robust multi-hop spatial reasoning in texts. Our experiments demonstrate that state-of-the-art models on the bAbI dataset struggle on the StepGame dataset. Moreover, we propose a Tensor-Product based Memory-Augmented Neural Network (TP-MANN) specialized for spatial reasoning tasks. Experimental results on both datasets show that our model outperforms all the baselines with superior generalization and robustness performance.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Inferring spatial relations in natural language is a crucial ability an intelligent system should possess. The bAbI dataset tries to capture tasks relevant to this domain (tasks 17 and 19). However, these tasks have several limitations. Most importantly, they are limited to fixed expressions, they are limited in the number of reasoning steps required to solve them, and they fail to test the robustness of models to input that contains irrelevant or redundant information. In this paper, we present a new Question-Answering dataset called StepGame for robust multi-hop spatial reasoning in texts. Our experiments demonstrate that state-of-the-art models on the bAbI dataset struggle on the StepGame dataset. Moreover, we propose a Tensor-Product based Memory-Augmented Neural Network (TP-MANN) specialized for spatial reasoning tasks. Experimental results on both datasets show that our model outperforms all the baselines with superior generalization and robustness performance. |
Shi, Zhengxiang; Feng, Yue; Lipani, Aldo Learning to Execute Actions or Ask Clarification Questions Inproceedings Findings of NAACL, 2022. Abstract | Links | BibTeX @inproceedings{shi-etal-2022-learning,
title = {Learning to Execute Actions or Ask Clarification Questions},
author = {Zhengxiang Shi and Yue Feng and Aldo Lipani},
url = {https://www.researchgate.net/publication/360050130_Learning_to_Execute_Actions_or_Ask_Clarification_Questions},
year = {2022},
date = {2022-01-01},
booktitle = {Findings of NAACL},
series = {NAACL '22},
abstract = {Collaborative tasks are ubiquitous activities where a form of communication is required in order to reach a joint goal. Collaborative building is one of such tasks. We wish to develop an intelligent builder agent in a simulated building environment (Minecraft) that can build whatever users wish to build by just talking to the agent. In order to achieve this goal, such agents need to be able to take the initiative by asking clarification questions when further information is needed. Existing works on Minecraft Corpus Dataset only learn to execute instructions neglecting the importance of asking for clarifications. In this paper, we extend the Minecraft Corpus Dataset by annotating all builder utterances into eight types, including clarification questions, and propose a new builder agent model capable of determining when to ask or execute instructions. Experimental results show that our model achieves state-of-the-art performance on the collabora-tive building task with a substantial improvement. We also define two new tasks, the learning to ask task and the joint learning task. The latter consists of solving both collaborating building and learning to ask tasks jointly.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Collaborative tasks are ubiquitous activities where a form of communication is required in order to reach a joint goal. Collaborative building is one of such tasks. We wish to develop an intelligent builder agent in a simulated building environment (Minecraft) that can build whatever users wish to build by just talking to the agent. In order to achieve this goal, such agents need to be able to take the initiative by asking clarification questions when further information is needed. Existing works on Minecraft Corpus Dataset only learn to execute instructions neglecting the importance of asking for clarifications. In this paper, we extend the Minecraft Corpus Dataset by annotating all builder utterances into eight types, including clarification questions, and propose a new builder agent model capable of determining when to ask or execute instructions. Experimental results show that our model achieves state-of-the-art performance on the collabora-tive building task with a substantial improvement. We also define two new tasks, the learning to ask task and the joint learning task. The latter consists of solving both collaborating building and learning to ask tasks jointly. |
2021
|
Rahmani, Hossein A; Yang, Jie Demographic Biases of Crowd Workers in Key Opinion Leaders Finding Workshop CSCW 2021 Workshop - Investigating and Mitigating Biases in Crowdsourced Data, 2021. Links | BibTeX @workshop{crowdbias2021rahmani,
title = {Demographic Biases of Crowd Workers in Key Opinion Leaders Finding},
author = {Hossein A. Rahmani and Jie Yang},
url = {http://arxiv.org/abs/2110.09248},
year = {2021},
date = {2021-10-30},
booktitle = {CSCW 2021 Workshop - Investigating and Mitigating Biases in Crowdsourced Data},
keywords = {},
pubstate = {published},
tppubtype = {workshop}
}
|
Zhang, Qiang; Fang, Jinyuan; Meng, Zaiqiao; Liang, Shangsong; Yilmaz, Emine Variational Continual Bayesian Meta-Learning Conference Thirty-Fifth Conference on Neural Information Processing Systems, 2021. Links | BibTeX @conference{zhang2021variational,
title = {Variational Continual Bayesian Meta-Learning},
author = {Qiang Zhang and Jinyuan Fang and Zaiqiao Meng and Shangsong Liang and Emine Yilmaz},
url = {https://openreview.net/forum?id=VH2og5jlrzm},
year = {2021},
date = {2021-10-20},
booktitle = {Thirty-Fifth Conference on Neural Information Processing Systems},
keywords = {},
pubstate = {published},
tppubtype = {conference}
}
|
Perez-Ortiz, Maria ; Dormann, Claire ; Rogers, Yvonne ; Bulathwela, Sahan ; Kreitmayer, Stefan ; Yilmaz, Emine ; Noss, Richard ; Shawe-Taylor, John X5Learn: A Personalised Learning Companion at the Intersection of AI and HCI Conference 26th International Conference on Intelligent User Interfaces, 2021. Links | BibTeX @conference{2021sahan,
title = {X5Learn: A Personalised Learning Companion at the Intersection of AI and HCI},
author = {Perez-Ortiz, Maria and Dormann, Claire and Rogers, Yvonne and Bulathwela, Sahan and Kreitmayer, Stefan and Yilmaz, Emine and Noss, Richard and Shawe-Taylor, John},
doi = {10.1145/3397482.3450721},
year = {2021},
date = {2021-04-28},
booktitle = {26th International Conference on Intelligent User Interfaces},
keywords = {},
pubstate = {published},
tppubtype = {conference}
}
|
Qiang Zhang; Aldo, Lipani; Emine Yilmaz Learning Neural Point Processes with Latent Graphs Inproceedings WWW, 2021. BibTeX @inproceedings{2021wwwqiang,
title = {Learning Neural Point Processes with Latent Graphs},
author = {Qiang Zhang; Aldo, Lipani; Emine Yilmaz},
year = {2021},
date = {2021-04-01},
booktitle = {WWW},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Ye, Fanghua; Manotumruksa, Jarana; Zhang, Qiang; Li, Shenghui; Yilmaz, Emine Slot Self-Attentive Dialogue State Tracking Inproceedings WWW, 2021. Links | BibTeX @inproceedings{www21ye,
title = {Slot Self-Attentive Dialogue State Tracking},
author = {Fanghua Ye and Jarana Manotumruksa and Qiang Zhang and Shenghui Li and Emine Yilmaz},
url = {https://arxiv.org/abs/2101.09374},
year = {2021},
date = {2021-04-01},
booktitle = {WWW},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Hofstätter, Sebastian; Lipani, Aldo; Althammer, Sophia; Zlabinger, Markus; Hanbury, Allan Mitigating the Position Bias of Transformer Models in Passage Re-Ranking Inproceedings ECIR, 2021. Links | BibTeX @inproceedings{2021ecir,
title = {Mitigating the Position Bias of Transformer Models in Passage Re-Ranking},
author = {Sebastian Hofstätter and Aldo Lipani and Sophia Althammer and Markus Zlabinger and Allan Hanbury},
url = {https://www.researchgate.net/publication/348589683_Mitigating_the_Position_Bias_of_Transformer_Models_in_Passage_Re-Ranking},
year = {2021},
date = {2021-03-28},
booktitle = {ECIR},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Gezici, Gizem; Lipani, Aldo; Saygin, Yucel; Yilmaz, Emine Evaluation metrics for measuring bias in search engine results Journal Article Information Retrieval Journal, 2021. Links | BibTeX @article{2021irj,
title = {Evaluation metrics for measuring bias in search engine results},
author = {Gizem Gezici and Aldo Lipani and Yucel Saygin and Emine Yilmaz },
doi = {https://doi.org/10.1007/s10791-020-09386-w},
year = {2021},
date = {2021-01-27},
journal = {Information Retrieval Journal},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
Lipani, Aldo; Carterette, Ben; Yilmaz, Emine How Am I Doing?: Evaluating Conversational Search Systems Offline Journal Article ACM Transactions on Information Systems (TOIS), 2021. Abstract | Links | BibTeX @article{Lipani2021TOIS,
title = {How Am I Doing?: Evaluating Conversational Search Systems Offline},
author = {Aldo Lipani and Ben Carterette and Emine Yilmaz},
url = {https://www.researchgate.net/publication/350640565_How_Am_I_Doing_Evaluating_Conversational_Search_Systems_Offline
https://aldolipani.com/wp-content/uploads/2021/04/How_Am_I_Doing-Evaluating_Conversational_Search_Systems_Offline.pdf},
year = {2021},
date = {2021-01-01},
journal = {ACM Transactions on Information Systems (TOIS)},
abstract = {As conversational agents like Siri and Alexa gain in popularity and use, conversation is becoming a more and more important mode of interaction for search. Conversational search shares some features with traditional search, but differs in some important respects: conversational search systems are less likely to return ranked lists of results (a SERP), more likely to involve iterated interactions, and more likely to feature longer, well-formed user queries in the form of natural language questions. Because of these differences, traditional methods for search evaluation (such as the Cranfield paradigm) do not translate easily to conversational search. In this work, we propose a framework for offline evaluation of conversational search, which includes a methodology for creating test collections with relevance judgments, an evaluation measure based on a user interaction model, and an approach to collecting user interaction data to train the model. The framework is based on the idea of “subtopics”, often used to model novelty and diversity in search and recommendation, and the user model is similar to the geometric browsing model introduced by RBP and used in ERR. As far as we know, this is the first work to combine these ideas into a comprehensive framework for offline evaluation of conversational search.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
As conversational agents like Siri and Alexa gain in popularity and use, conversation is becoming a more and more important mode of interaction for search. Conversational search shares some features with traditional search, but differs in some important respects: conversational search systems are less likely to return ranked lists of results (a SERP), more likely to involve iterated interactions, and more likely to feature longer, well-formed user queries in the form of natural language questions. Because of these differences, traditional methods for search evaluation (such as the Cranfield paradigm) do not translate easily to conversational search. In this work, we propose a framework for offline evaluation of conversational search, which includes a methodology for creating test collections with relevance judgments, an evaluation measure based on a user interaction model, and an approach to collecting user interaction data to train the model. The framework is based on the idea of “subtopics”, often used to model novelty and diversity in search and recommendation, and the user model is similar to the geometric browsing model introduced by RBP and used in ERR. As far as we know, this is the first work to combine these ideas into a comprehensive framework for offline evaluation of conversational search. |
Radmard, Puria; Fathullah, Yassir; Lipani, Aldo Subsequence Based Deep Active Learning for Named Entity Recognition Inproceedings Proceedings of the Association for Computational Linguistics, 2021. Links | BibTeX @inproceedings{Radmard2021,
title = {Subsequence Based Deep Active Learning for Named Entity Recognition},
author = {Puria Radmard and Yassir Fathullah and Aldo Lipani},
url = {https://www.researchgate.net/publication/351885762_Subsequence_Based_Deep_Active_Learning_for_Named_Entity_Recognition},
year = {2021},
date = {2021-01-01},
booktitle = {Proceedings of the Association for Computational Linguistics},
series = {ACL '21},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2020
|
Meng, Zaiqiao; McCreadie, Richard; Macdonald, Craig; Ounis, Iadh; Liu, Siwei; Wu, Yaxiong; Wang, Xi; Liang, Shangsong; Liang, Yucheng; Zeng, Guangtao; Liang, Junhua; Zhang, Qiang BETA-Rec: Build, Evaluate and Tune Automated Recommender Systems Inproceedings RecSys, 2020. BibTeX @inproceedings{2021recsys,
title = {BETA-Rec: Build, Evaluate and Tune Automated Recommender Systems},
author = {Zaiqiao Meng and Richard McCreadie and Craig Macdonald and Iadh Ounis and Siwei Liu and Yaxiong Wu and Xi Wang and Shangsong Liang and Yucheng Liang and Guangtao Zeng and Junhua Liang and Qiang Zhang},
year = {2020},
date = {2020-11-01},
booktitle = {RecSys},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Luo, Rui; Zhang, Qiang; Yang, Yaodong; Wang, Jun Replica-exchange Nosé-Hoover dynamics for Bayesian learning on large datasets Inproceedings NeurIPS, 2020. Links | BibTeX @inproceedings{NeurIPS2020,
title = {Replica-exchange Nosé-Hoover dynamics for Bayesian learning on large datasets},
author = {Rui Luo and Qiang Zhang and Yaodong Yang and Jun Wang},
url = {https://arxiv.org/abs/1905.12569},
year = {2020},
date = {2020-09-30},
booktitle = {NeurIPS},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Fanghua Ye; Jarana, Manotumruksa; Emine Yilmaz Unsupervised Few-Bits Semantic Hashing with Implicit Topics Modeling Inproceedings Findings of EMNLP, 2020. BibTeX @inproceedings{ye2020a,
title = {Unsupervised Few-Bits Semantic Hashing with Implicit Topics Modeling},
author = {Fanghua, Ye; Jarana, Manotumruksa; Emine, Yilmaz},
year = {2020},
date = {2020-09-01},
booktitle = {Findings of EMNLP},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Sebastian Hofstätter; Aldo, Lipani; Markus Zlabinger; Allan Hanbury Learning to Re-Rank with Contextualized Stopwords Inproceedings 2020. BibTeX @inproceedings{2020aa,
title = {Learning to Re-Rank with Contextualized Stopwords},
author = {Sebastian, Hofstätter; Aldo, Lipani; Markus, Zlabinger; Allan, Hanbury},
year = {2020},
date = {2020-07-07},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Qiang Zhang; Aldo, Lipani; Omer Kirnap; Emine Yilmaz Self-Attentive Hawkes Processes Inproceedings ICML, 2020. BibTeX @inproceedings{qiang2020a,
title = {Self-Attentive Hawkes Processes},
author = {Qiang ,Zhang; Aldo, Lipani; Omer, Kirnap; Emine Yilmaz},
year = {2020},
date = {2020-06-01},
booktitle = {ICML},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Sanchez Luis; He, Jiyin; Manotumruksa Jarana; Albakour Dyaa; Martinez Miguel; Lipani Aldo Easing Legal News Monitoring with Learning to Rank and BERT Inproceedings ECIR, 2020. Links | BibTeX @inproceedings{sanchez2020easing,
title = {Easing Legal News Monitoring with Learning to Rank and BERT},
author = {Sanchez, Luis; He, Jiyin; Manotumruksa, Jarana; Albakour, Dyaa; Martinez, Miguel; Lipani, Aldo},
doi = {10.1007/978-3-030-45442-5_42},
year = {2020},
date = {2020-04-08},
booktitle = {ECIR},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Bulathwela Sahan; Kreitmayer, Stefan; Perez-Ortiz Maria What's in it for me? Augmenting Recommended Learning Resources with Navigable Annotations Inproceedings Proceedings of the 25th International Conference on Intelligent User Interfaces Companion, pp. 114–115, 2020. Links | BibTeX @inproceedings{Bulathwela_iui_2020,
title = {What's in it for me? Augmenting Recommended Learning Resources with Navigable Annotations},
author = {Bulathwela, Sahan; Kreitmayer, Stefan; Perez-Ortiz, Maria},
doi = {10.1145/3379336.3381457},
year = {2020},
date = {2020-03-17},
booktitle = {Proceedings of the 25th International Conference on Intelligent User Interfaces Companion},
pages = {114--115},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Bulathwela Sahan; Perez-Ortiz, Maria; Yilmaz Emine; Shawe-Taylor John Towards an Integrative Educational Recommender for Lifelong Learners (Student Abstract) Inproceedings AAAI, 2020. Links | BibTeX @inproceedings{Bulathwela_aaai_2020b,
title = {Towards an Integrative Educational Recommender for Lifelong Learners (Student Abstract)},
author = {Bulathwela, Sahan; Perez-Ortiz, Maria; Yilmaz, Emine; Shawe-Taylor, John},
url = {https://aaai.org/ojs/index.php/AAAI/article/view/7151},
year = {2020},
date = {2020-02-07},
booktitle = {AAAI},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Bulathwela, Sahan; María Pérez-Ortiz; Rishabh Mehrotra; Davor Orlic; Colin De La Higuera; John Shawe-Taylor; Emine Yilmaz SUM'20: State-based User Modelling Inproceedings Proceedings of the 13th International Conference on Web Search and Data Mining, 2020. Links | BibTeX @inproceedings{Bulathwela_wsdm_2020,
title = {SUM'20: State-based User Modelling},
author = {Bulathwela, Sahan; María Pérez-Ortiz; Rishabh Mehrotra; Davor Orlic; Colin De La Higuera; John Shawe-Taylor; Emine Yilmaz},
doi = {10.1145/3336191.3371883},
year = {2020},
date = {2020-02-03},
booktitle = {Proceedings of the 13th International Conference on Web Search and Data Mining},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Sahan Bulathwela; Maria, Perez-Ortiz; Emine Yilmaz; John Shawe-Taylor Truelearn: A Family of Bayesian Algorithms to Match Lifelong Learners to Open Educational Resources Inproceedings AAAI, 2020. Links | BibTeX @inproceedings{sahan2020a,
title = {Truelearn: A Family of Bayesian Algorithms to Match Lifelong Learners to Open Educational Resources},
author = {Sahan, Bulathwela; Maria, Perez-Ortiz; Emine, Yilmaz; John, Shawe-Taylor},
doi = {10.1609/aaai.v34i01.5395},
year = {2020},
date = {2020-01-02},
booktitle = {AAAI},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Bulathwela Sahan; Perez-Ortiz, Maria; Lipani Aldo; Yilmaz Emine; Shawe-Taylor John Predicting Engagement in Video Lectures Inproceedings The Thirteenth International Conference on Educational Data Mining (EDM 2020), pp. 50–60, 2020. BibTeX @inproceedings{Bulathwela_edm_2020,
title = {Predicting Engagement in Video Lectures},
author = {Bulathwela, Sahan; Perez-Ortiz, Maria; Lipani, Aldo; Yilmaz, Emine; Shawe-Taylor, John},
year = {2020},
date = {2020-01-01},
booktitle = {The Thirteenth International Conference on Educational Data Mining (EDM 2020)},
pages = {50--60},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Molan, M; Bulathwela, S; Orlic, D Accessibility Recommendation Engine Inproceedings Proc.~of Int. Conf. on Open Educational Resources, 2020. BibTeX @inproceedings{2020asa,
title = {Accessibility Recommendation Engine},
author = {Molan, M. and Bulathwela, S. and Orlic, D.},
year = {2020},
date = {2020-01-01},
booktitle = {Proc.~of Int. Conf. on Open Educational Resources},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2019
|
Lipani, Aldo; Carterette, Ben; Yilmaz, Emine From a User Model for Query Sessions to Session Rank Biased Precision (sRBP) Inproceedings Proc.~of ICTIR, 2019. Abstract | Links | BibTeX @inproceedings{Lipani2019,
title = {From a User Model for Query Sessions to Session Rank Biased Precision (sRBP)},
author = {Aldo Lipani and Ben Carterette and Emine Yilmaz},
url = {https://www.researchgate.net/publication/334725760_From_a_User_Model_for_Query_Sessions_to_Session_Rank_Biased_Precision_sRBP},
doi = {10.1145/3341981.3344216},
year = {2019},
date = {2019-10-02},
booktitle = {Proc.~of ICTIR},
journal = {Proc.~of ICTIR},
abstract = {To satisfy their information needs, users usually carry out searches on retrieval systems by continuously trading off between the examination of search results retrieved by under-specified queries and the refinement of these queries through reformulation. In Information Retrieval (IR), a series of query reformulations is known as a query-session. Research in IR evaluation has traditionally been focused on the development of measures for the ad hoc task, for which a retrieval system aims to retrieve the best documents for a single query. Thus, most IR evaluation measures, with a few exceptions , are not suitable to evaluate retrieval scenarios that call for multiple refinements over a query-session. In this paper, by formally modeling a user's expected behaviour over query-sessions, we derive a session-based evaluation measure, which results in a generalization of the evaluation measure Rank Biased Precision (RBP). We demonstrate the quality of this new session-based evaluation measure, named Session RBP (sRBP), by evaluating its user model against the observed user behaviour over the query-sessions of the 2014 TREC Session track.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
To satisfy their information needs, users usually carry out searches on retrieval systems by continuously trading off between the examination of search results retrieved by under-specified queries and the refinement of these queries through reformulation. In Information Retrieval (IR), a series of query reformulations is known as a query-session. Research in IR evaluation has traditionally been focused on the development of measures for the ad hoc task, for which a retrieval system aims to retrieve the best documents for a single query. Thus, most IR evaluation measures, with a few exceptions , are not suitable to evaluate retrieval scenarios that call for multiple refinements over a query-session. In this paper, by formally modeling a user's expected behaviour over query-sessions, we derive a session-based evaluation measure, which results in a generalization of the evaluation measure Rank Biased Precision (RBP). We demonstrate the quality of this new session-based evaluation measure, named Session RBP (sRBP), by evaluating its user model against the observed user behaviour over the query-sessions of the 2014 TREC Session track. |
Shelbourne Charles; Linguaglossa, Leonardo; Lipani Aldo; Zhang Tianzhu; Geyer Fabien On the Learnability of Software Router Performance via CPU Measurements Inproceedings Proceedings of the 15th International Conference on emerging Networking EXperiments and Technologies, pp. 23–25, 2019. BibTeX @inproceedings{shelbourne2019learnability,
title = {On the Learnability of Software Router Performance via CPU Measurements},
author = {Shelbourne, Charles; Linguaglossa, Leonardo; Lipani, Aldo; Zhang, Tianzhu; Geyer, Fabien},
year = {2019},
date = {2019-08-01},
booktitle = {Proceedings of the 15th International Conference on emerging Networking EXperiments and Technologies},
pages = {23--25},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Liang, S; Yilmaz, E; Kanoulas, E Collaboratively Tracking Interests for User Clustering in Streams of Short Texts Journal Article IEEE Transactions on Knowledge and Data Engineering, 31 (2), pp. 257-272, 2019, ISSN: 1041-4347. Links | BibTeX @article{8355681,
title = {Collaboratively Tracking Interests for User Clustering in Streams of Short Texts},
author = {S Liang and E Yilmaz and E Kanoulas},
doi = {10.1109/TKDE.2018.2832211},
issn = {1041-4347},
year = {2019},
date = {2019-02-01},
journal = {IEEE Transactions on Knowledge and Data Engineering},
volume = {31},
number = {2},
pages = {257-272},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
Kreitmayer, Stefan ; Bulathwela, Sahan Future Learner Experiences with AI and Open Educational Resources Workshop Workshop on Towards a Responsible Innovation Agenda for HCI at ACM CHI Conference on Human Factors in Computing Systems, 2019. Links | BibTeX @workshop{2019sss,
title = {Future Learner Experiences with AI and Open Educational Resources},
author = {Kreitmayer, Stefan and Bulathwela, Sahan},
url = {http://wp.lancs.ac.uk/hci-responsible-innovation/files/2019/04/CHI2019_WS24_Final_Kreitmayer.pdf}},
year = {2019},
date = {2019-02-01},
booktitle = {Workshop on Towards a Responsible Innovation Agenda for HCI at ACM CHI Conference on Human Factors in Computing Systems},
keywords = {},
pubstate = {published},
tppubtype = {workshop}
}
|
Zhang, Qiang; Liang, Shangsong; Lipani, Aldo; Ren, Zhaochun; Yilmaz, Emine From Stances’ Imbalance to Their Hierarchical Representation and Detection Inproceedings Proc. of WWW, 2019. Links | BibTeX @inproceedings{Zhang2019bb,
title = {From Stances’ Imbalance to Their Hierarchical Representation and Detection},
author = {Qiang Zhang and Shangsong Liang and Aldo Lipani and Zhaochun Ren and Emine Yilmaz},
doi = {10.1145/3308558.3313724},
year = {2019},
date = {2019-01-01},
booktitle = {Proc. of WWW},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Zhang, Qiang; Lipani, Aldo; Liang, Shangsong; Yilmaz, Emine Reply-aided Detection of Misinformation via Bayesian Deep Learning Inproceedings Proc. of WWW, 2019. Links | BibTeX @inproceedings{Zhang2019bc,
title = {Reply-aided Detection of Misinformation via Bayesian Deep Learning},
author = {Qiang Zhang and Aldo Lipani and Shangsong Liang and Emine Yilmaz},
doi = {10.1145/3308558.3313718},
year = {2019},
date = {2019-01-01},
booktitle = {Proc. of WWW},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Sahan Bulathwela; Emine, Yilmaz; John Shawe-Taylor Towards Automatic, Scalable Quality Assurance in Open Education Inproceedings Workshop on Artificial Intelligence for SDG at International Joint Conference for Artificial Intelligence (IJCAI' 19), 2019. BibTeX @inproceedings{Bulathwela_ijcai_2019,
title = {Towards Automatic, Scalable Quality Assurance in Open Education},
author = {Sahan, Bulathwela; Emine, Yilmaz; John, Shawe-Taylor},
year = {2019},
date = {2019-01-01},
booktitle = {Workshop on Artificial Intelligence for SDG at International Joint Conference for Artificial Intelligence (IJCAI' 19)},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2018
|
Zhang, Qiang; Yilmaz, Emine; Liang, Shangsong Ranking-based Method for News Stance Detection Inproceedings Companion Proceedings of the The Web Conference 2018, pp. 41–42, International World Wide Web Conferences Steering Committee, Lyon, France, 2018, ISBN: 978-1-4503-5640-4. Links | BibTeX @inproceedings{Zhang:2018:RMN:3184558.3186919,
title = {Ranking-based Method for News Stance Detection},
author = {Qiang Zhang and Emine Yilmaz and Shangsong Liang},
url = {https://doi.org/10.1145/3184558.3186919},
doi = {10.1145/3184558.3186919},
isbn = {978-1-4503-5640-4},
year = {2018},
date = {2018-01-01},
booktitle = {Companion Proceedings of the The Web Conference 2018},
pages = {41--42},
publisher = {International World Wide Web Conferences Steering Committee},
address = {Lyon, France},
series = {WWW '18},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Mehrotra, Rishabh; Awadallah, Ahmed Hassan; Yilmaz, Emine LearnIR: WSDM 2018 Workshop on Learning from User Interactions Inproceedings Proceedings of the Eleventh ACM International Conference on Web Search and Data Mining, pp. 797–798, ACM, Marina Del Rey, CA, USA, 2018, ISBN: 978-1-4503-5581-0. Links | BibTeX @inproceedings{Mehrotra:2018:LWW:3159652.3160598,
title = {LearnIR: WSDM 2018 Workshop on Learning from User Interactions},
author = {Rishabh Mehrotra and Ahmed Hassan Awadallah and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/3159652.3160598},
doi = {10.1145/3159652.3160598},
isbn = {978-1-4503-5581-0},
year = {2018},
date = {2018-01-01},
booktitle = {Proceedings of the Eleventh ACM International Conference on Web Search and Data Mining},
pages = {797--798},
publisher = {ACM},
address = {Marina Del Rey, CA, USA},
series = {WSDM '18},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Verma, Manisha; Yilmaz, Emine; Craswell, Nick Study of Relevance and Effort Across Devices Inproceedings Proceedings of the 2018 Conference on Human Information Interaction & Retrieval, pp. 309–312, ACM, New Brunswick, NJ, USA, 2018, ISBN: 978-1-4503-4925-3. Links | BibTeX @inproceedings{Verma:2018:SRE:3176349.3176888,
title = {Study of Relevance and Effort Across Devices},
author = {Manisha Verma and Emine Yilmaz and Nick Craswell},
url = {http://doi.acm.org/10.1145/3176349.3176888},
doi = {10.1145/3176349.3176888},
isbn = {978-1-4503-4925-3},
year = {2018},
date = {2018-01-01},
booktitle = {Proceedings of the 2018 Conference on Human Information Interaction & Retrieval},
pages = {309--312},
publisher = {ACM},
address = {New Brunswick, NJ, USA},
series = {CHIIR '18},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Fang, Hui; Kamps, Jaap; Kanoulas, Evangelos; de Rijke, Maarten; Yilmaz, Emine Report on the 2017 ACM SIGIR International Conference Theory of Information Retrieval (ICTIR?17) Journal Article SIGIR Forum, 51 (3), pp. 78–87, 2018, ISSN: 0163-5840. Links | BibTeX @article{Fang:2018:RAS:3190580.3190591,
title = {Report on the 2017 ACM SIGIR International Conference Theory of Information Retrieval (ICTIR?17)},
author = {Hui Fang and Jaap Kamps and Evangelos Kanoulas and Maarten de Rijke and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/3190580.3190591},
doi = {10.1145/3190580.3190591},
issn = {0163-5840},
year = {2018},
date = {2018-01-01},
journal = {SIGIR Forum},
volume = {51},
number = {3},
pages = {78--87},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
2017
|
Liang, Shangsong; Ren, Zhaochun; Zhao, Yukun; Ma, Jun; Yilmaz, Emine; Rijke, Maarten De Inferring Dynamic User Interests in Streams of Short Texts for User Clustering Journal Article ACM Trans. Inf. Syst., 36 (1), pp. 10:1–10:37, 2017, ISSN: 1046-8188. Links | BibTeX @article{Liang:2017:IDU:3077622.3072606,
title = {Inferring Dynamic User Interests in Streams of Short Texts for User Clustering},
author = {Shangsong Liang and Zhaochun Ren and Yukun Zhao and Jun Ma and Emine Yilmaz and Maarten De Rijke},
url = {http://doi.acm.org/10.1145/3072606},
doi = {10.1145/3072606},
issn = {1046-8188},
year = {2017},
date = {2017-07-01},
journal = {ACM Trans. Inf. Syst.},
volume = {36},
number = {1},
pages = {10:1--10:37},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
Clarke, Charles L A; Yilmaz, Emine EVIA 2016: The Seventh International Workshop on Evaluating Information Access Journal Article SIGIR Forum, 50 (2), pp. 44–46, 2017, ISSN: 0163-5840. Links | BibTeX @article{Clarke:2017:ESI:3053408.3053418,
title = {EVIA 2016: The Seventh International Workshop on Evaluating Information Access},
author = {Charles L A Clarke and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/3053408.3053418},
doi = {10.1145/3053408.3053418},
issn = {0163-5840},
year = {2017},
date = {2017-02-01},
journal = {SIGIR Forum},
volume = {50},
number = {2},
pages = {44--46},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
He, Jiyin; Yilmaz, Emine User Behaviour and Task Characteristics: A Field Study of Daily Information Behaviour Inproceedings Best Paper Proceedings of the 2017 Conference on Conference Human Information Interaction and Retrieval, pp. 67–76, ACM, Oslo, Norway, 2017, ISBN: 978-1-4503-4677-1. Links | BibTeX @inproceedings{He:2017:UBT:3020165.3020188,
title = {User Behaviour and Task Characteristics: A Field Study of Daily Information Behaviour},
author = {Jiyin He and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/3020165.3020188},
doi = {10.1145/3020165.3020188},
isbn = {978-1-4503-4677-1},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 2017 Conference on Conference Human Information Interaction and Retrieval},
pages = {67--76},
publisher = {ACM},
address = {Oslo, Norway},
series = {CHIIR '17},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Mehrotra, Rishabh; Yilmaz, Emine Task Embeddings: Learning Query Embeddings Using Task Context Inproceedings Proceedings of the 2017 ACM on Conference on Information and Knowledge Management, pp. 2199–2202, ACM, Singapore, Singapore, 2017, ISBN: 978-1-4503-4918-5. Links | BibTeX @inproceedings{Mehrotra:2017:TEL:3132847.3133098,
title = {Task Embeddings: Learning Query Embeddings Using Task Context},
author = {Rishabh Mehrotra and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/3132847.3133098},
doi = {10.1145/3132847.3133098},
isbn = {978-1-4503-4918-5},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 2017 ACM on Conference on Information and Knowledge Management},
pages = {2199--2202},
publisher = {ACM},
address = {Singapore, Singapore},
series = {CIKM '17},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Mehrotra, Rishabh; Awadallah, Ahmed Hassan; Shokouhi, Milad; Yilmaz, Emine; Zitouni, Imed; Kholy, Ahmed El; Khabsa, Madian Deep Sequential Models for Task Satisfaction Prediction Inproceedings Proceedings of the 2017 ACM on Conference on Information and Knowledge Management, pp. 737–746, ACM, Singapore, Singapore, 2017, ISBN: 978-1-4503-4918-5. Links | BibTeX @inproceedings{Mehrotra:2017:DSM:3132847.3133001,
title = {Deep Sequential Models for Task Satisfaction Prediction},
author = {Rishabh Mehrotra and Ahmed Hassan Awadallah and Milad Shokouhi and Emine Yilmaz and Imed Zitouni and Ahmed El Kholy and Madian Khabsa},
url = {http://doi.acm.org/10.1145/3132847.3133001},
doi = {10.1145/3132847.3133001},
isbn = {978-1-4503-4918-5},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 2017 ACM on Conference on Information and Knowledge Management},
pages = {737--746},
publisher = {ACM},
address = {Singapore, Singapore},
series = {CIKM '17},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Mehrotra, Rishabh; Yilmaz, Emine Extracting Hierarchies of Search Tasks & Subtasks via a Bayesian Nonparametric Approach Inproceedings Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 285–294, ACM, Shinjuku, Tokyo, Japan, 2017, ISBN: 978-1-4503-5022-8. Links | BibTeX @inproceedings{Mehrotra:2017:EHS:3077136.3080823,
title = {Extracting Hierarchies of Search Tasks & Subtasks via a Bayesian Nonparametric Approach},
author = {Rishabh Mehrotra and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/3077136.3080823},
doi = {10.1145/3077136.3080823},
isbn = {978-1-4503-5022-8},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {285--294},
publisher = {ACM},
address = {Shinjuku, Tokyo, Japan},
series = {SIGIR '17},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Verma, Manisha; Yilmaz, Emine Search Costs vs. User Satisfaction on Mobile Inproceedings Jose, Joemon M; Hauff, Claudia; i, Ismail Sengor Alt; Song, Dawei; Albakour, Dyaa; Watt, Stuart; Tait, John (Ed.): pp. 698–704, Springer International Publishing, Cham, 2017, ISBN: 978-3-319-56608-5. Abstract | BibTeX @inproceedings{10.1007/978-3-319-56608-5_68,
title = {Search Costs vs. User Satisfaction on Mobile},
author = {Manisha Verma and Emine Yilmaz},
editor = {Joemon M Jose and Claudia Hauff and Ismail Sengor Alt{i}ngovde and Dawei Song and Dyaa Albakour and Stuart Watt and John Tait},
isbn = {978-3-319-56608-5},
year = {2017},
date = {2017-01-01},
pages = {698--704},
publisher = {Springer International Publishing},
address = {Cham},
abstract = {Information seeking is an interactive process where users submit search queries, read snippets or click on documents until their information need is satisfied. User cost-benefit models have recently gained popularity to study search behaviour. These models assume that a user gains information at expense of some cost. Primary assumption is that an adept user would maximize gain while minimizing search costs. However, existing work only provides an estimate of user cost or benefit per action, it does not explore how these costs are correlated with user satisfaction. Moreover, parameters of these models are determined by desktop based observational studies. Whether these parameters vary with device is unknown. In this paper we address both problems by studying how these models correlate with user satisfaction and determine parameters on data collected via mobile based search study. Our experiments indicate that several parameters indeed differ in mobile setting and that existing cost functions, when applied to mobile search, do not highly correlate with user satisfaction.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Information seeking is an interactive process where users submit search queries, read snippets or click on documents until their information need is satisfied. User cost-benefit models have recently gained popularity to study search behaviour. These models assume that a user gains information at expense of some cost. Primary assumption is that an adept user would maximize gain while minimizing search costs. However, existing work only provides an estimate of user cost or benefit per action, it does not explore how these costs are correlated with user satisfaction. Moreover, parameters of these models are determined by desktop based observational studies. Whether these parameters vary with device is unknown. In this paper we address both problems by studying how these models correlate with user satisfaction and determine parameters on data collected via mobile based search study. Our experiments indicate that several parameters indeed differ in mobile setting and that existing cost functions, when applied to mobile search, do not highly correlate with user satisfaction. |
Mehrotra, Rishabh; Anderson, Ashton; Diaz, Fernando; Sharma, Amit; Wallach, Hanna; Yilmaz, Emine Auditing Search Engines for Differential Satisfaction Across Demographics Inproceedings Proceedings of the 26th International Conference on World Wide Web Companion, pp. 626–633, International World Wide Web Conferences Steering Committee, Perth, Australia, 2017, ISBN: 978-1-4503-4914-7. Links | BibTeX @inproceedings{Mehrotra:2017:ASE:3041021.3054197,
title = {Auditing Search Engines for Differential Satisfaction Across Demographics},
author = {Rishabh Mehrotra and Ashton Anderson and Fernando Diaz and Amit Sharma and Hanna Wallach and Emine Yilmaz},
url = {https://doi.org/10.1145/3041021.3054197},
doi = {10.1145/3041021.3054197},
isbn = {978-1-4503-4914-7},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 26th International Conference on World Wide Web Companion},
pages = {626--633},
publisher = {International World Wide Web Conferences Steering Committee},
address = {Perth, Australia},
series = {WWW '17 Companion},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Zou, Bin; Lampos, Vasileios; Liang, Shangsong; Ren, Zhaochun; Yilmaz, Emine; Cox, Ingemar A Concept Language Model for Ad-hoc Retrieval Inproceedings Proceedings of the 26th International Conference on World Wide Web Companion, pp. 885–886, International World Wide Web Conferences Steering Committee, Perth, Australia, 2017, ISBN: 978-1-4503-4914-7. Links | BibTeX @inproceedings{Zou:2017:CLM:3041021.3054209,
title = {A Concept Language Model for Ad-hoc Retrieval},
author = {Bin Zou and Vasileios Lampos and Shangsong Liang and Zhaochun Ren and Emine Yilmaz and Ingemar Cox},
url = {https://doi.org/10.1145/3041021.3054209},
doi = {10.1145/3041021.3054209},
isbn = {978-1-4503-4914-7},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the 26th International Conference on World Wide Web Companion},
pages = {885--886},
publisher = {International World Wide Web Conferences Steering Committee},
address = {Perth, Australia},
series = {WWW '17 Companion},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Liang, Shangsong; Ren, Zhaochun; Yilmaz, Emine; Kanoulas, Evangelos Collaborative User Clustering for Short Text Streams Inproceedings Proceedings of the Thirty-First AAAI Conference on Artificial Intelligence,
February 4-9, 2017, San Francisco, California, USA., pp. 3504–3510, 2017. Links | BibTeX @inproceedings{DBLP:conf/aaai/LiangRYK17,
title = {Collaborative User Clustering for Short Text Streams},
author = {Shangsong Liang and Zhaochun Ren and Emine Yilmaz and Evangelos Kanoulas},
url = {http://aaai.org/ocs/index.php/AAAI/AAAI17/paper/view/14792},
year = {2017},
date = {2017-01-01},
booktitle = {Proceedings of the Thirty-First AAAI Conference on Artificial Intelligence,
February 4-9, 2017, San Francisco, California, USA.},
pages = {3504--3510},
crossref = {DBLP:conf/aaai/2017},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Liang, Shangsong; Yilmaz, Emine; Shen, Hong; Rijke, Maarten De; Croft, Bruce W Search Result Diversification in Short Text Streams Journal Article ACM Trans. Inf. Syst., 36 (1), pp. 8:1–8:35, 2017, ISSN: 1046-8188. Links | BibTeX @article{Liang:2017:SRD:3077622.3057282,
title = {Search Result Diversification in Short Text Streams},
author = {Shangsong Liang and Emine Yilmaz and Hong Shen and Maarten De Rijke and Bruce W Croft},
url = {http://doi.acm.org/10.1145/3057282},
doi = {10.1145/3057282},
issn = {1046-8188},
year = {2017},
date = {2017-01-01},
journal = {ACM Trans. Inf. Syst.},
volume = {36},
number = {1},
pages = {8:1--8:35},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
Koolen, Marijn; Kamps, Jaap; Bogers, Toine; Belkin, Nicholas; Kelly, Diane; Yilmaz, Emine Report on the Second Workshop on Supporting Complex Search Tasks Journal Article SIGIR Forum, 51 (1), pp. 58–66, 2017, ISSN: 0163-5840. Links | BibTeX @article{Koolen:2017:RSW:3130332.3130343,
title = {Report on the Second Workshop on Supporting Complex Search Tasks},
author = {Marijn Koolen and Jaap Kamps and Toine Bogers and Nicholas Belkin and Diane Kelly and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/3130332.3130343},
doi = {10.1145/3130332.3130343},
issn = {0163-5840},
year = {2017},
date = {2017-01-01},
journal = {SIGIR Forum},
volume = {51},
number = {1},
pages = {58--66},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
2016
|
Mehrotra, Rishabh; Bhattacharya, Prasanta; Yilmaz, Emine Deconstructing Complex Search Tasks: a Bayesian Nonparametric Approach for Extracting Sub-tasks Inproceedings Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 599–605, Association for Computational Linguistics, San Diego, California, 2016. Links | BibTeX @inproceedings{mehrotra-bhattacharya-yilmaz:2016:N16-1,
title = {Deconstructing Complex Search Tasks: a Bayesian Nonparametric Approach for Extracting Sub-tasks},
author = {Rishabh Mehrotra and Prasanta Bhattacharya and Emine Yilmaz},
url = {http://www.aclweb.org/anthology/N16-1073},
year = {2016},
date = {2016-06-01},
booktitle = {Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
pages = {599--605},
publisher = {Association for Computational Linguistics},
address = {San Diego, California},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Liang, Shangsong; Yilmaz, Emine; Kanoulas, Evangelos Dynamic Clustering of Streaming Short Documents Inproceedings Proceedings of the 22Nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 995–1004, ACM, San Francisco, California, USA, 2016, ISBN: 978-1-4503-4232-2. Links | BibTeX @inproceedings{Liang:2016:DCS:2939672.2939748,
title = {Dynamic Clustering of Streaming Short Documents},
author = {Shangsong Liang and Emine Yilmaz and Evangelos Kanoulas},
url = {http://doi.acm.org/10.1145/2939672.2939748},
doi = {10.1145/2939672.2939748},
isbn = {978-1-4503-4232-2},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 22Nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining},
pages = {995--1004},
publisher = {ACM},
address = {San Francisco, California, USA},
series = {KDD '16},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Zhao, Yukun; Liang, Shangsong; Ren, Zhaochun; Ma, Jun; Yilmaz, Emine; de Rijke, Maarten Explainable User Clustering in Short Text Streams Inproceedings Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 155–164, ACM, Pisa, Italy, 2016, ISBN: 978-1-4503-4069-4. Links | BibTeX @inproceedings{Zhao:2016:EUC:2911451.2911522,
title = {Explainable User Clustering in Short Text Streams},
author = {Yukun Zhao and Shangsong Liang and Zhaochun Ren and Jun Ma and Emine Yilmaz and Maarten de Rijke},
url = {http://doi.acm.org/10.1145/2911451.2911522},
doi = {10.1145/2911451.2911522},
isbn = {978-1-4503-4069-4},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {155--164},
publisher = {ACM},
address = {Pisa, Italy},
series = {SIGIR '16},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Mehrotra, Rishabh; Bhattacharya, Prasanta; Yilmaz, Emine Uncovering Task Based Behavioral Heterogeneities in Online Search Behavior Inproceedings Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1049–1052, ACM, Pisa, Italy, 2016, ISBN: 978-1-4503-4069-4. Links | BibTeX @inproceedings{Mehrotra:2016:UTB:2911451.2914755,
title = {Uncovering Task Based Behavioral Heterogeneities in Online Search Behavior},
author = {Rishabh Mehrotra and Prasanta Bhattacharya and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2911451.2914755},
doi = {10.1145/2911451.2914755},
isbn = {978-1-4503-4069-4},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {1049--1052},
publisher = {ACM},
address = {Pisa, Italy},
series = {SIGIR '16},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Zhang, Dell; Wang, Jun; Yilmaz, Emine; Wang, Xiaoling; Zhou, Yuxin Bayesian Performance Comparison of Text Classifiers Inproceedings Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 15–24, ACM, Pisa, Italy, 2016, ISBN: 978-1-4503-4069-4. Links | BibTeX @inproceedings{Zhang:2016:BPC:2911451.2911547,
title = {Bayesian Performance Comparison of Text Classifiers},
author = {Dell Zhang and Jun Wang and Emine Yilmaz and Xiaoling Wang and Yuxin Zhou},
url = {http://doi.acm.org/10.1145/2911451.2911547},
doi = {10.1145/2911451.2911547},
isbn = {978-1-4503-4069-4},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {15--24},
publisher = {ACM},
address = {Pisa, Italy},
series = {SIGIR '16},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Verma, Manisha; Yilmaz, Emine Characterizing Relevance on Mobile and Desktop Inproceedings Ferro, Nicola; Crestani, Fabio; Moens, Marie-Francine; Mothe, Josiane; Silvestri, Fabrizio; Nunzio, Giorgio Maria Di; Hauff, Claudia; Silvello, Gianmaria (Ed.): pp. 212–223, Springer International Publishing, Cham, 2016, ISBN: 978-3-319-30671-1. Abstract | BibTeX @inproceedings{10.1007/978-3-319-30671-1_16,
title = {Characterizing Relevance on Mobile and Desktop},
author = {Manisha Verma and Emine Yilmaz},
editor = {Nicola Ferro and Fabio Crestani and Marie-Francine Moens and Josiane Mothe and Fabrizio Silvestri and Giorgio Maria Di Nunzio and Claudia Hauff and Gianmaria Silvello},
isbn = {978-3-319-30671-1},
year = {2016},
date = {2016-01-01},
pages = {212--223},
publisher = {Springer International Publishing},
address = {Cham},
abstract = {Relevance judgments are central to Information retrieval evaluation. With increasing number of hand held devices at users disposal today, and continuous improvement in web standards and browsers, it has become essential to evaluate whether such devices and dynamic page layouts affect users notion of relevance. Given dynamic web pages and content rendering, we know little about what kind of pages are relevant on devices other than desktop. With this work, we take the first step in characterizing relevance on mobiles and desktop. We collect crowd sourced judgments on mobile and desktop to systematically determine whether screen size of a device and page layouts impact judgments. Our study shows that there are certain difference between mobile and desktop judgments. We also observe different judging times, despite similar inter-rater agreement on both devices. Finally, we also propose and evaluate display and viewport specific features to predict relevance. Our results indicate that viewport based features can be used to reliably predict mobile relevance.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Relevance judgments are central to Information retrieval evaluation. With increasing number of hand held devices at users disposal today, and continuous improvement in web standards and browsers, it has become essential to evaluate whether such devices and dynamic page layouts affect users notion of relevance. Given dynamic web pages and content rendering, we know little about what kind of pages are relevant on devices other than desktop. With this work, we take the first step in characterizing relevance on mobiles and desktop. We collect crowd sourced judgments on mobile and desktop to systematically determine whether screen size of a device and page layouts impact judgments. Our study shows that there are certain difference between mobile and desktop judgments. We also observe different judging times, despite similar inter-rater agreement on both devices. Finally, we also propose and evaluate display and viewport specific features to predict relevance. Our results indicate that viewport based features can be used to reliably predict mobile relevance. |
Mehrotra, Rishabh; Bhattacharya, Prasanta; Yilmaz, Emine Characterizing Users' Multi-Tasking Behavior in Web Search Inproceedings Proceedings of the 2016 ACM on Conference on Human Information Interaction and Retrieval, pp. 297–300, ACM, Carrboro, North Carolina, USA, 2016, ISBN: 978-1-4503-3751-9. Links | BibTeX @inproceedings{Mehrotra:2016:CUM:2854946.2855006,
title = {Characterizing Users' Multi-Tasking Behavior in Web Search},
author = {Rishabh Mehrotra and Prasanta Bhattacharya and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2854946.2855006},
doi = {10.1145/2854946.2855006},
isbn = {978-1-4503-3751-9},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 2016 ACM on Conference on Human Information Interaction and Retrieval},
pages = {297--300},
publisher = {ACM},
address = {Carrboro, North Carolina, USA},
series = {CHIIR '16},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Verma, Manisha; Yilmaz, Emine Category Oriented Task Extraction Inproceedings Proceedings of the 2016 ACM on Conference on Human Information Interaction and Retrieval, pp. 333–336, ACM, Carrboro, North Carolina, USA, 2016, ISBN: 978-1-4503-3751-9. Links | BibTeX @inproceedings{Verma:2016:COT:2854946.2854997,
title = {Category Oriented Task Extraction},
author = {Manisha Verma and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2854946.2854997},
doi = {10.1145/2854946.2854997},
isbn = {978-1-4503-3751-9},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the 2016 ACM on Conference on Human Information Interaction and Retrieval},
pages = {333--336},
publisher = {ACM},
address = {Carrboro, North Carolina, USA},
series = {CHIIR '16},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Verma, Manisha; Yilmaz, Emine; Craswell, Nick On Obtaining Effort Based Judgements for Information Retrieval Inproceedings Proceedings of the Ninth ACM International Conference on Web Search and Data Mining, pp. 277–286, ACM, San Francisco, California, USA, 2016, ISBN: 978-1-4503-3716-8. Links | BibTeX @inproceedings{Verma:2016:OEB:2835776.2835840,
title = {On Obtaining Effort Based Judgements for Information Retrieval},
author = {Manisha Verma and Emine Yilmaz and Nick Craswell},
url = {http://doi.acm.org/10.1145/2835776.2835840},
doi = {10.1145/2835776.2835840},
isbn = {978-1-4503-3716-8},
year = {2016},
date = {2016-01-01},
booktitle = {Proceedings of the Ninth ACM International Conference on Web Search and Data Mining},
pages = {277--286},
publisher = {ACM},
address = {San Francisco, California, USA},
series = {WSDM '16},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2015
|
Yilmaz, Emine; Verma, Manisha; Mehrotra, Rishabh; Kanoulas, Evangelos; Carterette, Ben; Craswell, Nick Overview of the TREC 2015 Tasks Track. Inproceedings TREC, 2015. BibTeX @inproceedings{yilmaz2015overview,
title = {Overview of the TREC 2015 Tasks Track.},
author = {Emine Yilmaz and Manisha Verma and Rishabh Mehrotra and Evangelos Kanoulas and Ben Carterette and Nick Craswell},
year = {2015},
date = {2015-01-01},
booktitle = {TREC},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Mehrotra, Rishabh; Yilmaz, Emine Terms, Topics & Tasks: Enhanced User Modelling for Better Personalization Inproceedings Proceedings of the 2015 International Conference on The Theory of Information Retrieval, pp. 131–140, ACM, Northampton, Massachusetts, USA, 2015, ISBN: 978-1-4503-3833-2. Links | BibTeX @inproceedings{Mehrotra:2015:TTT:2808194.2809467,
title = {Terms, Topics & Tasks: Enhanced User Modelling for Better Personalization},
author = {Rishabh Mehrotra and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2808194.2809467},
doi = {10.1145/2808194.2809467},
isbn = {978-1-4503-3833-2},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the 2015 International Conference on The Theory of Information Retrieval},
pages = {131--140},
publisher = {ACM},
address = {Northampton, Massachusetts, USA},
series = {ICTIR '15},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Mehrotra, Rishabh; Yilmaz, Emine Representative & Informative Query Selection for Learning to Rank Using Submodular Functions Inproceedings Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 545–554, ACM, Santiago, Chile, 2015, ISBN: 978-1-4503-3621-5. Links | BibTeX @inproceedings{Mehrotra:2015:RIQ:2766462.2767753,
title = {Representative & Informative Query Selection for Learning to Rank Using Submodular Functions},
author = {Rishabh Mehrotra and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2766462.2767753},
doi = {10.1145/2766462.2767753},
isbn = {978-1-4503-3621-5},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {545--554},
publisher = {ACM},
address = {Santiago, Chile},
series = {SIGIR '15},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Shokouhi, Milad; White, Ryen; Yilmaz, Emine Anchoring and Adjustment in Relevance Estimation Inproceedings Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 963–966, ACM, Santiago, Chile, 2015, ISBN: 978-1-4503-3621-5. Links | BibTeX @inproceedings{Shokouhi:2015:AAR:2766462.2767841,
title = {Anchoring and Adjustment in Relevance Estimation},
author = {Milad Shokouhi and Ryen White and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2766462.2767841},
doi = {10.1145/2766462.2767841},
isbn = {978-1-4503-3621-5},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {963--966},
publisher = {ACM},
address = {Santiago, Chile},
series = {SIGIR '15},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Mehrotra, Rishabh; Yilmaz, Emine Towards Hierarchies of Search Tasks & Subtasks Inproceedings Proceedings of the 24th International Conference on World Wide Web, pp. 73–74, ACM, Florence, Italy, 2015, ISBN: 978-1-4503-3473-0. Links | BibTeX @inproceedings{Mehrotra:2015:THS:2740908.2742777,
title = {Towards Hierarchies of Search Tasks & Subtasks},
author = {Rishabh Mehrotra and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2740908.2742777},
doi = {10.1145/2740908.2742777},
isbn = {978-1-4503-3473-0},
year = {2015},
date = {2015-01-01},
booktitle = {Proceedings of the 24th International Conference on World Wide Web},
pages = {73--74},
publisher = {ACM},
address = {Florence, Italy},
series = {WWW '15 Companion},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2014
|
Yilmaz, Emine; Kanoulas, Evangelos; Craswell, Nick Effect of Intent Descriptions on Retrieval Evaluation Inproceedings Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management, pp. 599–608, ACM, Shanghai, China, 2014, ISBN: 978-1-4503-2598-1. Links | BibTeX @inproceedings{Yilmaz:2014:EID:2661829.2661950,
title = {Effect of Intent Descriptions on Retrieval Evaluation},
author = {Emine Yilmaz and Evangelos Kanoulas and Nick Craswell},
url = {http://doi.acm.org/10.1145/2661829.2661950},
doi = {10.1145/2661829.2661950},
isbn = {978-1-4503-2598-1},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management},
pages = {599--608},
publisher = {ACM},
address = {Shanghai, China},
series = {CIKM '14},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Yilmaz, Emine; Verma, Manisha; Craswell, Nick; Radlinski, Filip; Bailey, Peter Relevance and Effort: An Analysis of Document Utility Inproceedings Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management, pp. 91–100, ACM, Shanghai, China, 2014, ISBN: 978-1-4503-2598-1. Links | BibTeX @inproceedings{Yilmaz:2014:REA:2661829.2661953,
title = {Relevance and Effort: An Analysis of Document Utility},
author = {Emine Yilmaz and Manisha Verma and Nick Craswell and Filip Radlinski and Peter Bailey},
url = {http://doi.acm.org/10.1145/2661829.2661953},
doi = {10.1145/2661829.2661953},
isbn = {978-1-4503-2598-1},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management},
pages = {91--100},
publisher = {ACM},
address = {Shanghai, China},
series = {CIKM '14},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Verma, Manisha; Yilmaz, Emine Entity Oriented Task Extraction from Query Logs Inproceedings Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management, pp. 1975–1978, ACM, Shanghai, China, 2014, ISBN: 978-1-4503-2598-1. Links | BibTeX @inproceedings{Verma:2014:EOT:2661829.2662076,
title = {Entity Oriented Task Extraction from Query Logs},
author = {Manisha Verma and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2661829.2662076},
doi = {10.1145/2661829.2662076},
isbn = {978-1-4503-2598-1},
year = {2014},
date = {2014-01-01},
booktitle = {Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management},
pages = {1975--1978},
publisher = {ACM},
address = {Shanghai, China},
series = {CIKM '14},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Mehrotra, Rishabh; Yilmaz, Emine; Verma, Manisha Task-Based User Modelling for Personalization via Probabilistic Matrix Factorization. Inproceedings RecSys Posters, 2014. BibTeX @inproceedings{mehrotra2014task,
title = {Task-Based User Modelling for Personalization via Probabilistic Matrix Factorization.},
author = {Rishabh Mehrotra and Emine Yilmaz and Manisha Verma},
year = {2014},
date = {2014-01-01},
booktitle = {RecSys Posters},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2013
|
Kazai, Gabriella; Yilmaz, Emine; Craswell, Nick; Tahaghoghi, S M M User Intent and Assessor Disagreement in Web Search Evaluation Inproceedings Proceedings of the 22Nd ACM International Conference on Information & Knowledge Management, pp. 699–708, ACM, San Francisco, California, USA, 2013, ISBN: 978-1-4503-2263-8. Links | BibTeX @inproceedings{Kazai:2013:UIA:2505515.2505716,
title = {User Intent and Assessor Disagreement in Web Search Evaluation},
author = {Gabriella Kazai and Emine Yilmaz and Nick Craswell and S M M Tahaghoghi},
url = {http://doi.acm.org/10.1145/2505515.2505716},
doi = {10.1145/2505515.2505716},
isbn = {978-1-4503-2263-8},
year = {2013},
date = {2013-01-01},
booktitle = {Proceedings of the 22Nd ACM International Conference on Information & Knowledge Management},
pages = {699--708},
publisher = {ACM},
address = {San Francisco, California, USA},
series = {CIKM '13},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Robertson, Stephen; Kanoulas, Evangelos; Yilmaz, Emine Modelling Score Distributions Without Actual Scores Inproceedings Proceedings of the 2013 Conference on the Theory of Information Retrieval, pp. 20:85–20:92, ACM, Copenhagen, Denmark, 2013, ISBN: 978-1-4503-2107-5. Links | BibTeX @inproceedings{Robertson:2013:MSD:2499178.2499181,
title = {Modelling Score Distributions Without Actual Scores},
author = {Stephen Robertson and Evangelos Kanoulas and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2499178.2499181},
doi = {10.1145/2499178.2499181},
isbn = {978-1-4503-2107-5},
year = {2013},
date = {2013-01-01},
booktitle = {Proceedings of the 2013 Conference on the Theory of Information Retrieval},
pages = {20:85--20:92},
publisher = {ACM},
address = {Copenhagen, Denmark},
series = {ICTIR '13},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Lease, Matthew; Yilmaz, Emine Crowdsourcing for information retrieval: introduction to the special issue Journal Article Information Retrieval, 16 (2), pp. 91–100, 2013, ISSN: 1573-7659. Abstract | Links | BibTeX @article{Lease2013,
title = {Crowdsourcing for information retrieval: introduction to the special issue},
author = {Matthew Lease and Emine Yilmaz},
url = {https://doi.org/10.1007/s10791-013-9222-7},
doi = {10.1007/s10791-013-9222-7},
issn = {1573-7659},
year = {2013},
date = {2013-01-01},
journal = {Information Retrieval},
volume = {16},
number = {2},
pages = {91--100},
abstract = {This introduction to the special issue summarizes and contextualizes six novel research contributions at the intersection of information retrieval (IR) and crowdsourcing (also overlapping crowdsourcing's closely-related sibling, human computation). Several of the papers included in this special issue represent deeper investigations into research topics for which earlier stages of the authors' research were disseminated at crowdsourcing workshops at SIGIR and WSDM conferences, as well as at the NIST TREC conference. Since the first proposed use of crowdsourcing for IR in 2008, interest in this area has quickly accelerated and led to three workshops, an ongoing NIST TREC track, and a great variety of published papers, talks, and tutorials. We briefly summarize the area in order to help situate the contributions appearing in this special issue. We also discuss some broader current trends and issues in crowdsourcing which bear upon its use in IR and other fields.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
This introduction to the special issue summarizes and contextualizes six novel research contributions at the intersection of information retrieval (IR) and crowdsourcing (also overlapping crowdsourcing's closely-related sibling, human computation). Several of the papers included in this special issue represent deeper investigations into research topics for which earlier stages of the authors' research were disseminated at crowdsourcing workshops at SIGIR and WSDM conferences, as well as at the NIST TREC conference. Since the first proposed use of crowdsourcing for IR in 2008, interest in this area has quickly accelerated and led to three workshops, an ongoing NIST TREC track, and a great variety of published papers, talks, and tutorials. We briefly summarize the area in order to help situate the contributions appearing in this special issue. We also discuss some broader current trends and issues in crowdsourcing which bear upon its use in IR and other fields. |
Clarke, Charles L A; Freund, Luanne; Smucker, Mark D; Yilmaz, Emine Report on the SIGIR 2013 Workshop on Modeling User Behavior for Information Retrieval Evaluation (MUBE 2013) Journal Article SIGIR Forum, 47 (2), pp. 84–95, 2013, ISSN: 0163-5840. Links | BibTeX @article{Clarke:2013:RSW:2568388.2568403,
title = {Report on the SIGIR 2013 Workshop on Modeling User Behavior for Information Retrieval Evaluation (MUBE 2013)},
author = {Charles L A Clarke and Luanne Freund and Mark D Smucker and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2568388.2568403},
doi = {10.1145/2568388.2568403},
issn = {0163-5840},
year = {2013},
date = {2013-01-01},
journal = {SIGIR Forum},
volume = {47},
number = {2},
pages = {84--95},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
Zuccon, Guido; Leelanupab, Teerapong; Whiting, Stewart; Yilmaz, Emine; Jose, Joemon M; Azzopardi, Leif Crowdsourcing Interactions: Using Crowdsourcing for Evaluating Interactive Information Retrieval Systems Journal Article Inf. Retr., 16 (2), pp. 267–305, 2013, ISSN: 1386-4564. Links | BibTeX @article{Zuccon:2013:CIU:2460567.2460569,
title = {Crowdsourcing Interactions: Using Crowdsourcing for Evaluating Interactive Information Retrieval Systems},
author = {Guido Zuccon and Teerapong Leelanupab and Stewart Whiting and Emine Yilmaz and Joemon M Jose and Leif Azzopardi},
url = {http://dx.doi.org/10.1007/s10791-012-9206-z},
doi = {10.1007/s10791-012-9206-z},
issn = {1386-4564},
year = {2013},
date = {2013-01-01},
journal = {Inf. Retr.},
volume = {16},
number = {2},
pages = {267--305},
publisher = {Kluwer Academic Publishers},
address = {Hingham, MA, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
2012
|
Carterette, Ben; Kanoulas, Evangelos; Yilmaz, Emine Incorporating Variability in User Behavior into Systems Based Evaluation Inproceedings Proceedings of the 21st ACM International Conference on Information and Knowledge Management, pp. 135–144, ACM, Maui, Hawaii, USA, 2012, ISBN: 978-1-4503-1156-4. Links | BibTeX @inproceedings{Carterette:2012:IVU:2396761.2396782,
title = {Incorporating Variability in User Behavior into Systems Based Evaluation},
author = {Ben Carterette and Evangelos Kanoulas and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2396761.2396782},
doi = {10.1145/2396761.2396782},
isbn = {978-1-4503-1156-4},
year = {2012},
date = {2012-01-01},
booktitle = {Proceedings of the 21st ACM International Conference on Information and Knowledge Management},
pages = {135--144},
publisher = {ACM},
address = {Maui, Hawaii, USA},
series = {CIKM '12},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Kazai, Gabriella; Craswell, Nick; Yilmaz, Emine; Tahaghoghi, S M M An Analysis of Systematic Judging Errors in Information Retrieval Inproceedings Proceedings of the 21st ACM International Conference on Information and Knowledge Management, pp. 105–114, ACM, Maui, Hawaii, USA, 2012, ISBN: 978-1-4503-1156-4. Links | BibTeX @inproceedings{Kazai:2012:ASJ:2396761.2396779,
title = {An Analysis of Systematic Judging Errors in Information Retrieval},
author = {Gabriella Kazai and Nick Craswell and Emine Yilmaz and S M M Tahaghoghi},
url = {http://doi.acm.org/10.1145/2396761.2396779},
doi = {10.1145/2396761.2396779},
isbn = {978-1-4503-1156-4},
year = {2012},
date = {2012-01-01},
booktitle = {Proceedings of the 21st ACM International Conference on Information and Knowledge Management},
pages = {105--114},
publisher = {ACM},
address = {Maui, Hawaii, USA},
series = {CIKM '12},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Yilmaz, Emine; Kazai, Gabriella; Craswell, Nick; Tahaghoghi, Saied Mehrizi On Judgments Obtained from a Commercial Search Engine Inproceedings Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1115–1116, ACM, Portland, Oregon, USA, 2012, ISBN: 978-1-4503-1472-5. Links | BibTeX @inproceedings{Yilmaz:2012:JOC:2348283.2348496,
title = {On Judgments Obtained from a Commercial Search Engine},
author = {Emine Yilmaz and Gabriella Kazai and Nick Craswell and Saied Mehrizi Tahaghoghi},
url = {http://doi.acm.org/10.1145/2348283.2348496},
doi = {10.1145/2348283.2348496},
isbn = {978-1-4503-1472-5},
year = {2012},
date = {2012-01-01},
booktitle = {Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {1115--1116},
publisher = {ACM},
address = {Portland, Oregon, USA},
series = {SIGIR '12},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Hosseini, Mehdi; Cox, Ingemar J; Milic-Frayling, Natasa; Shokouhi, Milad; Yilmaz, Emine An Uncertainty-aware Query Selection Model for Evaluation of IR Systems Inproceedings Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 901–910, ACM, Portland, Oregon, USA, 2012, ISBN: 978-1-4503-1472-5. Links | BibTeX @inproceedings{Hosseini:2012:UQS:2348283.2348403,
title = {An Uncertainty-aware Query Selection Model for Evaluation of IR Systems},
author = {Mehdi Hosseini and Ingemar J Cox and Natasa Milic-Frayling and Milad Shokouhi and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2348283.2348403},
doi = {10.1145/2348283.2348403},
isbn = {978-1-4503-1472-5},
year = {2012},
date = {2012-01-01},
booktitle = {Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {901--910},
publisher = {ACM},
address = {Portland, Oregon, USA},
series = {SIGIR '12},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Lease, Matthew; Yilmaz, Emine Crowdsourcing for Information Retrieval Journal Article SIGIR Forum, 45 (2), pp. 66–75, 2012, ISSN: 0163-5840. Links | BibTeX @article{Lease:2012:CIR:2093346.2093356,
title = {Crowdsourcing for Information Retrieval},
author = {Matthew Lease and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2093346.2093356},
doi = {10.1145/2093346.2093356},
issn = {0163-5840},
year = {2012},
date = {2012-01-01},
journal = {SIGIR Forum},
volume = {45},
number = {2},
pages = {66--75},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
2011
|
Szummer, Martin; Yilmaz, Emine Semi-supervised Learning to Rank with Preference Regularization Inproceedings Proceedings of the 20th ACM International Conference on Information and Knowledge Management, pp. 269–278, ACM, Glasgow, Scotland, UK, 2011, ISBN: 978-1-4503-0717-8. Links | BibTeX @inproceedings{Szummer:2011:SLR:2063576.2063620,
title = {Semi-supervised Learning to Rank with Preference Regularization},
author = {Martin Szummer and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2063576.2063620},
doi = {10.1145/2063576.2063620},
isbn = {978-1-4503-0717-8},
year = {2011},
date = {2011-01-01},
booktitle = {Proceedings of the 20th ACM International Conference on Information and Knowledge Management},
pages = {269--278},
publisher = {ACM},
address = {Glasgow, Scotland, UK},
series = {CIKM '11},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Carterette, Ben; Kanoulas, Evangelos; Yilmaz, Emine Simulating Simple User Behavior for System Effectiveness Evaluation Inproceedings Proceedings of the 20th ACM International Conference on Information and Knowledge Management, pp. 611–620, ACM, Glasgow, Scotland, UK, 2011, ISBN: 978-1-4503-0717-8. Links | BibTeX @inproceedings{Carterette:2011:SSU:2063576.2063668,
title = {Simulating Simple User Behavior for System Effectiveness Evaluation},
author = {Ben Carterette and Evangelos Kanoulas and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2063576.2063668},
doi = {10.1145/2063576.2063668},
isbn = {978-1-4503-0717-8},
year = {2011},
date = {2011-01-01},
booktitle = {Proceedings of the 20th ACM International Conference on Information and Knowledge Management},
pages = {611--620},
publisher = {ACM},
address = {Glasgow, Scotland, UK},
series = {CIKM '11},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Wang, Chang; Yilmaz, Emine; Szummer, Martin Relevance Feedback Exploiting Query-specific Document Manifolds Inproceedings Proceedings of the 20th ACM International Conference on Information and Knowledge Management, pp. 1957–1960, ACM, Glasgow, Scotland, UK, 2011, ISBN: 978-1-4503-0717-8. Links | BibTeX @inproceedings{Wang:2011:RFE:2063576.2063864,
title = {Relevance Feedback Exploiting Query-specific Document Manifolds},
author = {Chang Wang and Emine Yilmaz and Martin Szummer},
url = {http://doi.acm.org/10.1145/2063576.2063864},
doi = {10.1145/2063576.2063864},
isbn = {978-1-4503-0717-8},
year = {2011},
date = {2011-01-01},
booktitle = {Proceedings of the 20th ACM International Conference on Information and Knowledge Management},
pages = {1957--1960},
publisher = {ACM},
address = {Glasgow, Scotland, UK},
series = {CIKM '11},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Bennett, Paul N; Radlinski, Filip; White, Ryen W; Yilmaz, Emine Inferring and Using Location Metadata to Personalize Web Search Inproceedings Proceedings of the 34th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 135–144, ACM, Beijing, China, 2011, ISBN: 978-1-4503-0757-4. Links | BibTeX @inproceedings{Bennett:2011:IUL:2009916.2009938,
title = {Inferring and Using Location Metadata to Personalize Web Search},
author = {Paul N Bennett and Filip Radlinski and Ryen W White and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/2009916.2009938},
doi = {10.1145/2009916.2009938},
isbn = {978-1-4503-0757-4},
year = {2011},
date = {2011-01-01},
booktitle = {Proceedings of the 34th International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {135--144},
publisher = {ACM},
address = {Beijing, China},
series = {SIGIR '11},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Radlinski, Filip; Bennett, Paul N; Yilmaz, Emine Detecting Duplicate Web Documents Using Clickthrough Data Inproceedings Proceedings of the Fourth ACM International Conference on Web Search and Data Mining, pp. 147–156, ACM, Hong Kong, China, 2011, ISBN: 978-1-4503-0493-1. Links | BibTeX @inproceedings{Radlinski:2011:DDW:1935826.1935859,
title = {Detecting Duplicate Web Documents Using Clickthrough Data},
author = {Filip Radlinski and Paul N Bennett and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1935826.1935859},
doi = {10.1145/1935826.1935859},
isbn = {978-1-4503-0493-1},
year = {2011},
date = {2011-01-01},
booktitle = {Proceedings of the Fourth ACM International Conference on Web Search and Data Mining},
pages = {147--156},
publisher = {ACM},
address = {Hong Kong, China},
series = {WSDM '11},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Lease, Matthew; Carvalho, Vitor R; Yilmaz, Emine Crowdsourcing for Search and Data Mining Journal Article SIGIR Forum, 45 (1), pp. 18–24, 2011, ISSN: 0163-5840. Links | BibTeX @article{Lease:2011:CSD:1988852.1988856,
title = {Crowdsourcing for Search and Data Mining},
author = {Matthew Lease and Vitor R Carvalho and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1988852.1988856},
doi = {10.1145/1988852.1988856},
issn = {0163-5840},
year = {2011},
date = {2011-01-01},
journal = {SIGIR Forum},
volume = {45},
number = {1},
pages = {18--24},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
Carvalho, Vitor R; Lease, Matthew; Yilmaz, Emine Crowdsourcing for Search Evaluation Journal Article SIGIR Forum, 44 (2), pp. 17–22, 2011, ISSN: 0163-5840. Links | BibTeX @article{Carvalho:2011:CSE:1924475.1924481,
title = {Crowdsourcing for Search Evaluation},
author = {Vitor R Carvalho and Matthew Lease and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1924475.1924481},
doi = {10.1145/1924475.1924481},
issn = {0163-5840},
year = {2011},
date = {2011-01-01},
journal = {SIGIR Forum},
volume = {44},
number = {2},
pages = {17--22},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
2010
|
Yilmaz, Emine; Robertson, Stephen On the choice of effectiveness measures for learning to rank Journal Article Information Retrieval, 13 (3), pp. 271–290, 2010, ISSN: 1573-7659. Abstract | Links | BibTeX @article{Yilmaz2010,
title = {On the choice of effectiveness measures for learning to rank},
author = {Emine Yilmaz and Stephen Robertson},
url = {https://doi.org/10.1007/s10791-009-9116-x},
doi = {10.1007/s10791-009-9116-x},
issn = {1573-7659},
year = {2010},
date = {2010-06-01},
journal = {Information Retrieval},
volume = {13},
number = {3},
pages = {271--290},
abstract = {Most current machine learning methods for building search engines are based on the assumption that there is a target evaluation metric that evaluates the quality of the search engine with respect to an end user and the engine should be trained to optimize for that metric. Treating the target evaluation metric as a given, many different approaches (e.g. LambdaRank, SoftRank, RankingSVM, etc.) have been proposed to develop methods for optimizing for retrieval metrics. Target metrics used in optimization act as bottlenecks that summarize the training data and it is known that some evaluation metrics are more informative than others. In this paper, we consider the effect of the target evaluation metric on learning to rank. In particular, we question the current assumption that retrieval systems should be designed to directly optimize for a metric that is assumed to evaluate user satisfaction. We show that even if user satisfaction can be measured by a metric X, optimizing the engine on a training set for a more informative metric Y may result in a better test performance according to X (as compared to optimizing the engine directly for X on the training set). We analyze the situations as to when there is a significant difference in the two cases in terms of the amount of available training data and the number of dimensions of the feature space.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Most current machine learning methods for building search engines are based on the assumption that there is a target evaluation metric that evaluates the quality of the search engine with respect to an end user and the engine should be trained to optimize for that metric. Treating the target evaluation metric as a given, many different approaches (e.g. LambdaRank, SoftRank, RankingSVM, etc.) have been proposed to develop methods for optimizing for retrieval metrics. Target metrics used in optimization act as bottlenecks that summarize the training data and it is known that some evaluation metrics are more informative than others. In this paper, we consider the effect of the target evaluation metric on learning to rank. In particular, we question the current assumption that retrieval systems should be designed to directly optimize for a metric that is assumed to evaluate user satisfaction. We show that even if user satisfaction can be measured by a metric X, optimizing the engine on a training set for a more informative metric Y may result in a better test performance according to X (as compared to optimizing the engine directly for X on the training set). We analyze the situations as to when there is a significant difference in the two cases in terms of the amount of available training data and the number of dimensions of the feature space. |
Yilmaz, Emine; Shokouhi, Milad; Craswell, Nick; Robertson, Stephen Expected Browsing Utility for Web Search Evaluation Inproceedings Proceedings of the 19th ACM International Conference on Information and Knowledge Management, pp. 1561–1564, ACM, Toronto, ON, Canada, 2010, ISBN: 978-1-4503-0099-5. Links | BibTeX @inproceedings{Yilmaz:2010:EBU:1871437.1871672,
title = {Expected Browsing Utility for Web Search Evaluation},
author = {Emine Yilmaz and Milad Shokouhi and Nick Craswell and Stephen Robertson},
url = {http://doi.acm.org/10.1145/1871437.1871672},
doi = {10.1145/1871437.1871672},
isbn = {978-1-4503-0099-5},
year = {2010},
date = {2010-01-01},
booktitle = {Proceedings of the 19th ACM International Conference on Information and Knowledge Management},
pages = {1561--1564},
publisher = {ACM},
address = {Toronto, ON, Canada},
series = {CIKM '10},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Carterette, Ben; Kanoulas, Evangelos; Yilmaz, Emine Low Cost Evaluation in Information Retrieval Inproceedings Proceedings of the 33rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 903–903, ACM, Geneva, Switzerland, 2010, ISBN: 978-1-4503-0153-4. Links | BibTeX @inproceedings{Carterette:2010:LCE:1835449.1835675,
title = {Low Cost Evaluation in Information Retrieval},
author = {Ben Carterette and Evangelos Kanoulas and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1835449.1835675},
doi = {10.1145/1835449.1835675},
isbn = {978-1-4503-0153-4},
year = {2010},
date = {2010-01-01},
booktitle = {Proceedings of the 33rd International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {903--903},
publisher = {ACM},
address = {Geneva, Switzerland},
series = {SIGIR '10},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Robertson, Stephen E; Kanoulas, Evangelos; Yilmaz, Emine Extending Average Precision to Graded Relevance Judgments Inproceedings Proceedings of the 33rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 603–610, ACM, Geneva, Switzerland, 2010, ISBN: 978-1-4503-0153-4. Links | BibTeX @inproceedings{Robertson:2010:EAP:1835449.1835550,
title = {Extending Average Precision to Graded Relevance Judgments},
author = {Stephen E Robertson and Evangelos Kanoulas and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1835449.1835550},
doi = {10.1145/1835449.1835550},
isbn = {978-1-4503-0153-4},
year = {2010},
date = {2010-01-01},
booktitle = {Proceedings of the 33rd International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {603--610},
publisher = {ACM},
address = {Geneva, Switzerland},
series = {SIGIR '10},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2009
|
Craswell, Nick; Fetterly, Dennis; Najork, Marc; Robertson, Stephen; Yilmaz, Emine Microsoft Research at TREC 2009: Web and relevance feedback tracks Journal Article 2009. BibTeX @article{craswell2009microsoft,
title = {Microsoft Research at TREC 2009: Web and relevance feedback tracks},
author = {Nick Craswell and Dennis Fetterly and Marc Najork and Stephen Robertson and Emine Yilmaz},
year = {2009},
date = {2009-01-01},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
Yilmaz, Emine; Robertson, Stephen Deep Versus Shallow Judgments in Learning to Rank Inproceedings Proceedings of the 32Nd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 662–663, ACM, Boston, MA, USA, 2009, ISBN: 978-1-60558-483-6. Links | BibTeX @inproceedings{Yilmaz:2009:DVS:1571941.1572066,
title = {Deep Versus Shallow Judgments in Learning to Rank},
author = {Emine Yilmaz and Stephen Robertson},
url = {http://doi.acm.org/10.1145/1571941.1572066},
doi = {10.1145/1571941.1572066},
isbn = {978-1-60558-483-6},
year = {2009},
date = {2009-01-01},
booktitle = {Proceedings of the 32Nd International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {662--663},
publisher = {ACM},
address = {Boston, MA, USA},
series = {SIGIR '09},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Aslam, Javed A; Kanoulas, Evangelos; Pavlu, Virgil; Savev, Stefan; Yilmaz, Emine Document Selection Methodologies for Efficient and Effective Learning-to-rank Inproceedings Proceedings of the 32Nd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 468–475, ACM, Boston, MA, USA, 2009, ISBN: 978-1-60558-483-6. Links | BibTeX @inproceedings{Aslam:2009:DSM:1571941.1572022,
title = {Document Selection Methodologies for Efficient and Effective Learning-to-rank},
author = {Javed A Aslam and Evangelos Kanoulas and Virgil Pavlu and Stefan Savev and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1571941.1572022},
doi = {10.1145/1571941.1572022},
isbn = {978-1-60558-483-6},
year = {2009},
date = {2009-01-01},
booktitle = {Proceedings of the 32Nd International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {468--475},
publisher = {ACM},
address = {Boston, MA, USA},
series = {SIGIR '09},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2008
|
Bailey, Peter; Craswell, Nick; Soboroff, Ian; Thomas, Paul; de Vries, Arjen P; Yilmaz, Emine Relevance Assessment: Are Judges Exchangeable and Does It Matter Inproceedings Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 667–674, ACM, Singapore, Singapore, 2008, ISBN: 978-1-60558-164-4. Links | BibTeX @inproceedings{Bailey:2008:RAJ:1390334.1390447,
title = {Relevance Assessment: Are Judges Exchangeable and Does It Matter},
author = {Peter Bailey and Nick Craswell and Ian Soboroff and Paul Thomas and Arjen P de Vries and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1390334.1390447},
doi = {10.1145/1390334.1390447},
isbn = {978-1-60558-164-4},
year = {2008},
date = {2008-01-01},
booktitle = {Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {667--674},
publisher = {ACM},
address = {Singapore, Singapore},
series = {SIGIR '08},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Yilmaz, Emine; Aslam, Javed A; Robertson, Stephen A New Rank Correlation Coefficient for Information Retrieval Inproceedings Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 587–594, ACM, Singapore, Singapore, 2008, ISBN: 978-1-60558-164-4. Links | BibTeX @inproceedings{Yilmaz:2008:NRC:1390334.1390435,
title = {A New Rank Correlation Coefficient for Information Retrieval},
author = {Emine Yilmaz and Javed A Aslam and Stephen Robertson},
url = {http://doi.acm.org/10.1145/1390334.1390435},
doi = {10.1145/1390334.1390435},
isbn = {978-1-60558-164-4},
year = {2008},
date = {2008-01-01},
booktitle = {Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {587--594},
publisher = {ACM},
address = {Singapore, Singapore},
series = {SIGIR '08},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Yilmaz, Emine; Kanoulas, Evangelos; Aslam, Javed A A Simple and Efficient Sampling Method for Estimating AP and NDCG Inproceedings Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 603–610, ACM, Singapore, Singapore, 2008, ISBN: 978-1-60558-164-4. Links | BibTeX @inproceedings{Yilmaz:2008:SES:1390334.1390437,
title = {A Simple and Efficient Sampling Method for Estimating AP and NDCG},
author = {Emine Yilmaz and Evangelos Kanoulas and Javed A Aslam},
url = {http://doi.acm.org/10.1145/1390334.1390437},
doi = {10.1145/1390334.1390437},
isbn = {978-1-60558-164-4},
year = {2008},
date = {2008-01-01},
booktitle = {Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {603--610},
publisher = {ACM},
address = {Singapore, Singapore},
series = {SIGIR '08},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Yilmaz, Emine; Aslam, Javed A Estimating average precision when judgments are incomplete Journal Article Knowledge and Information Systems, 16 (2), pp. 173–211, 2008, ISSN: 0219-3116. Abstract | Links | BibTeX @article{Yilmaz2008,
title = {Estimating average precision when judgments are incomplete},
author = {Emine Yilmaz and Javed A Aslam},
url = {https://doi.org/10.1007/s10115-007-0101-7},
doi = {10.1007/s10115-007-0101-7},
issn = {0219-3116},
year = {2008},
date = {2008-01-01},
journal = {Knowledge and Information Systems},
volume = {16},
number = {2},
pages = {173--211},
abstract = {We consider the problem of evaluating retrieval systems with incomplete relevance judgments. Recently, Buckley and Voorhees showed that standard measures of retrieval performance are not robust to incomplete judgments, and they proposed a new measure, bpref, that is much more robust to incomplete judgments. Although bpref is highly correlated with average precision when the judgments are effectively complete, the value of bpref deviates from average precision and from its own value as the judgment set degrades, especially at very low levels of assessment. In this work, we propose three new evaluation measures induced AP, subcollection AP, and inferred AP that are equivalent to average precision when the relevance judgments are complete and that are statistical estimates of average precision when relevance judgments are a random subset of complete judgments. We consider natural scenarios which yield highly incomplete judgments such as random judgment sets or very shallow depth pools. We compare and contrast the robustness of the three measures proposed in this work with bpref for both of these scenarios. Through the use of TREC data, we demonstrate that these measures are more robust to incomplete relevance judgments than bpref, both in terms of how well the measures estimate average precision (as measured with complete relevance judgments) and how well they estimate themselves (as measured with complete relevance judgments). Finally, since inferred AP is the most accurate approximation to average precision and the most robust measure in the presence of incomplete judgments, we provide a detailed analysis of this measure, both in terms of its behavior in theory and its implementation in practice.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
We consider the problem of evaluating retrieval systems with incomplete relevance judgments. Recently, Buckley and Voorhees showed that standard measures of retrieval performance are not robust to incomplete judgments, and they proposed a new measure, bpref, that is much more robust to incomplete judgments. Although bpref is highly correlated with average precision when the judgments are effectively complete, the value of bpref deviates from average precision and from its own value as the judgment set degrades, especially at very low levels of assessment. In this work, we propose three new evaluation measures induced AP, subcollection AP, and inferred AP that are equivalent to average precision when the relevance judgments are complete and that are statistical estimates of average precision when relevance judgments are a random subset of complete judgments. We consider natural scenarios which yield highly incomplete judgments such as random judgment sets or very shallow depth pools. We compare and contrast the robustness of the three measures proposed in this work with bpref for both of these scenarios. Through the use of TREC data, we demonstrate that these measures are more robust to incomplete relevance judgments than bpref, both in terms of how well the measures estimate average precision (as measured with complete relevance judgments) and how well they estimate themselves (as measured with complete relevance judgments). Finally, since inferred AP is the most accurate approximation to average precision and the most robust measure in the presence of incomplete judgments, we provide a detailed analysis of this measure, both in terms of its behavior in theory and its implementation in practice. |
2007
|
Aslam, Javed A; Yilmaz, Emine Inferring Document Relevance from Incomplete Information Inproceedings Proceedings of the Sixteenth ACM Conference on Conference on Information and Knowledge Management, pp. 633–642, ACM, Lisbon, Portugal, 2007, ISBN: 978-1-59593-803-9. Links | BibTeX @inproceedings{Aslam:2007:IDR:1321440.1321529,
title = {Inferring Document Relevance from Incomplete Information},
author = {Javed A Aslam and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1321440.1321529},
doi = {10.1145/1321440.1321529},
isbn = {978-1-59593-803-9},
year = {2007},
date = {2007-01-01},
booktitle = {Proceedings of the Sixteenth ACM Conference on Conference on Information and Knowledge Management},
pages = {633--642},
publisher = {ACM},
address = {Lisbon, Portugal},
series = {CIKM '07},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2006
|
Yilmaz, Emine; Aslam, Javed A Estimating Average Precision with Incomplete and Imperfect Judgments Inproceedings Proceedings of the 15th ACM International Conference on Information and Knowledge Management, pp. 102–111, ACM, Arlington, Virginia, USA, 2006, ISBN: 1-59593-433-2. Links | BibTeX @inproceedings{Yilmaz:2006:EAP:1183614.1183633,
title = {Estimating Average Precision with Incomplete and Imperfect Judgments},
author = {Emine Yilmaz and Javed A Aslam},
url = {http://doi.acm.org/10.1145/1183614.1183633},
doi = {10.1145/1183614.1183633},
isbn = {1-59593-433-2},
year = {2006},
date = {2006-01-01},
booktitle = {Proceedings of the 15th ACM International Conference on Information and Knowledge Management},
pages = {102--111},
publisher = {ACM},
address = {Arlington, Virginia, USA},
series = {CIKM '06},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Aslam, Javed A; Pavlu, Virgil; Yilmaz, Emine A Statistical Method for System Evaluation Using Incomplete Judgments Inproceedings Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 541–548, ACM, Seattle, Washington, USA, 2006, ISBN: 1-59593-369-7. Links | BibTeX @inproceedings{Aslam:2006:SMS:1148170.1148263,
title = {A Statistical Method for System Evaluation Using Incomplete Judgments},
author = {Javed A Aslam and Virgil Pavlu and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1148170.1148263},
doi = {10.1145/1148170.1148263},
isbn = {1-59593-369-7},
year = {2006},
date = {2006-01-01},
booktitle = {Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {541--548},
publisher = {ACM},
address = {Seattle, Washington, USA},
series = {SIGIR '06},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Aslam, Javed A; Yilmaz, Emine Inferring Document Relevance via Average Precision Inproceedings Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 601–602, ACM, Seattle, Washington, USA, 2006, ISBN: 1-59593-369-7. Links | BibTeX @inproceedings{Aslam:2006:IDR:1148170.1148275,
title = {Inferring Document Relevance via Average Precision},
author = {Javed A Aslam and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1148170.1148275},
doi = {10.1145/1148170.1148275},
isbn = {1-59593-369-7},
year = {2006},
date = {2006-01-01},
booktitle = {Proceedings of the 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {601--602},
publisher = {ACM},
address = {Seattle, Washington, USA},
series = {SIGIR '06},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
2005
|
Aslam, Javed A; Yilmaz, Emine A Geometric Interpretation and Analysis of R-precision Inproceedings Proceedings of the 14th ACM International Conference on Information and Knowledge Management, pp. 664–671, ACM, Bremen, Germany, 2005, ISBN: 1-59593-140-6. Links | BibTeX @inproceedings{Aslam:2005:GIA:1099554.1099721,
title = {A Geometric Interpretation and Analysis of R-precision},
author = {Javed A Aslam and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1099554.1099721},
doi = {10.1145/1099554.1099721},
isbn = {1-59593-140-6},
year = {2005},
date = {2005-01-01},
booktitle = {Proceedings of the 14th ACM International Conference on Information and Knowledge Management},
pages = {664--671},
publisher = {ACM},
address = {Bremen, Germany},
series = {CIKM '05},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Aslam, Javed A; Yilmaz, Emine; Pavlu, Virgiliu The Maximum Entropy Method for Analyzing Retrieval Measures Inproceedings Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 27–34, ACM, Salvador, Brazil, 2005, ISBN: 1-59593-034-5. Links | BibTeX @inproceedings{Aslam:2005:MEM:1076034.1076042,
title = {The Maximum Entropy Method for Analyzing Retrieval Measures},
author = {Javed A Aslam and Emine Yilmaz and Virgiliu Pavlu},
url = {http://doi.acm.org/10.1145/1076034.1076042},
doi = {10.1145/1076034.1076042},
isbn = {1-59593-034-5},
year = {2005},
date = {2005-01-01},
booktitle = {Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {27--34},
publisher = {ACM},
address = {Salvador, Brazil},
series = {SIGIR '05},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Aslam, Javed A; Yilmaz, Emine; Pavlu, Virgiliu A Geometric Interpretation of R-precision and Its Correlation with Average Precision Inproceedings Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 573–574, ACM, Salvador, Brazil, 2005, ISBN: 1-59593-034-5. Links | BibTeX @inproceedings{Aslam:2005:GIR:1076034.1076134,
title = {A Geometric Interpretation of R-precision and Its Correlation with Average Precision},
author = {Javed A Aslam and Emine Yilmaz and Virgiliu Pavlu},
url = {http://doi.acm.org/10.1145/1076034.1076134},
doi = {10.1145/1076034.1076134},
isbn = {1-59593-034-5},
year = {2005},
date = {2005-01-01},
booktitle = {Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {573--574},
publisher = {ACM},
address = {Salvador, Brazil},
series = {SIGIR '05},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Aslam, Javed A; Pavlu, Virgiliu; Yilmaz, Emine Measure-based Metasearch Inproceedings Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 571–572, ACM, Salvador, Brazil, 2005, ISBN: 1-59593-034-5. Links | BibTeX @inproceedings{Aslam:2005:MM:1076034.1076133,
title = {Measure-based Metasearch},
author = {Javed A Aslam and Virgiliu Pavlu and Emine Yilmaz},
url = {http://doi.acm.org/10.1145/1076034.1076133},
doi = {10.1145/1076034.1076133},
isbn = {1-59593-034-5},
year = {2005},
date = {2005-01-01},
booktitle = {Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval},
pages = {571--572},
publisher = {ACM},
address = {Salvador, Brazil},
series = {SIGIR '05},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
0000
|
Inproceedings 0000. BibTeX @inproceedings{Zhang2019b,
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|
Ramin Okhrati; Aldo, Lipani A Multilinear Sampling Algorithm to Estimate Shapley Values Inproceedings 0000. BibTeX @inproceedings{2020a,
title = {A Multilinear Sampling Algorithm to Estimate Shapley Values},
author = {Ramin, Okhrati; Aldo, Lipani},
series = {ICPR},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
|