@inproceedings{688a122624e74065b873993dec2dcfac,
title = "Crossflow: A framework for distributed mining of software repositories",
abstract = "Large-scale software repository mining typically requires substantial storage and computational resources, and often involves a large number of calls to (rate-limited) APIs such as those of GitHub and StackOverflow. This creates a growing need for distributed execution of repository mining programs to which remote collaborators can contribute computational and storage resources, as well as API quotas (ideally without sharing API access tokens or credentials). In this paper we introduce Crossflow, a novel framework for building distributed repository mining programs. We demonstrate how Crossflow can delegate mining jobs to remote workers and cache their results, and how workers can implement advanced behaviour such as load balancing and rejecting jobs they cannot perform (e.g. due to lack of space, credentials for a specific API).",
keywords = "Client-server systems, Computer aided software engineering, Data analysis, Data collection, Data flow computing, Data integration, Distributed processing, Modeling, Open source software, Pipeline pro cessing, Public domain software, Scalability, Software engineering",
author = "Dimitris Kolovos and Patrick Neubauer and Konstantinos Barmpis and Nicholas Matragkas and Richard Paige",
year = "2019",
month = may,
day = "1",
doi = "10.1109/MSR.2019.00032",
language = "English",
series = "IEEE International Working Conference on Mining Software Repositories",
publisher = "IEEE Computer Society",
pages = "155--159",
booktitle = "Proceedings - 2019 IEEE/ACM 16th International Conference on Mining Software Repositories, MSR 2019",
address = "United States",
note = "16th IEEE/ACM International Conference on Mining Software Repositories, MSR 2019 ; Conference date: 26-05-2019 Through 27-05-2019",
}