@article {Krull2020,
	title = {Artificial-intelligence-driven scanning probe microscopy},
	journal = {Communications Physics},
	volume = {3},
	year = {2020},
	abstract = {Scanning probe microscopy (SPM) has revolutionized the fields of materials, nano-science, chemistry, and biology, by enabling mapping of surface properties and surface manipulation with atomic precision. However, these achievements require constant human supervision; fully automated SPM has not been accomplished yet. Here we demonstrate an artificial intelligence framework based on machine learning for autonomous SPM operation (DeepSPM). DeepSPM includes an algorithmic search of good sample regions, a convolutional neural network to assess the quality of acquired images, and a deep reinforcement learning agent to reliably condition the state of the probe. DeepSPM is able to acquire and classify data continuously in multi-day scanning tunneling microscopy experiments, managing the probe quality in response to varying experimental conditions. Our approach paves the way for advanced methods hardly feasible by human operation (e.g., large dataset acquisition and SPM-based nanolithography). DeepSPM can be generalized to most SPM techniques, with the source code publicly available.},
	issn = {23993650},
	doi = {10.1038/s42005-020-0317-3},
	author = {Krull, A. and Hirsch, P. and Rother, C. and Schiffrin, A. and Krull, C.}
}
@article {Alhaija2018,
	title = {Augmented Reality Meets Computer Vision},
	journal = {International Journal of Computer Vision},
	volume = {In press},
	year = {2018},
	pages = {1{\textendash}13},
	abstract = {The success of deep learning in computer vision is based on availability of large annotated datasets. To lower the need for hand labeled images, virtually rendered 3D worlds have recently gained popularity. Creating realistic 3D content is challenging on its own and requires significant human effort. In this work, we propose an alternative paradigm which combines real and synthetic data for learning semantic instance segmentation and object detection models. Exploiting the fact that not all aspects of the scene are equally important for this task, we propose to augment real-world imagery with virtual objects of the target category. Capturing real-world images at large scale is easy and cheap, and directly provides real background appearances without the need for creating complex 3D models of the environment. We present an efficient procedure to augment real images with virtual objects. This allows us to create realistic composite images which exhibit both realistic background appearance and a large number of complex object arrangements. In contrast to modeling complete 3D environments, our augmentation approach requires only a few user interactions in combination with 3D shapes of the target object. Through extensive experimentation, we conclude the right set of parameters to produce augmented data which can maximally enhance the performance of instance segmentation models. Further, we demonstrate the utility of our approach on training standard deep models for semantic instance segmentation and object detection of cars in outdoor driving scenes. We test the models trained on our augmented data on the KITTI 2015 dataset, which we have annotated with pixel-accurate ground truth, and on Cityscapes dataset. Our experiments demonstrate that models trained on augmented imagery generalize better than those trained on synthetic data or models trained on limited amount of annotated real data.},
	keywords = {autonomous driving, data augmenta-, instance segmentation, synthetic training data, tion},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Mescheder, Lars and Geiger, Andreas and Carsten Rother}
}
@article {6268,
	title = {Augmented Reality Meets Computer Vision Efficient Data Generation for Urban Driving Scenes},
	journal = {IJCV},
	year = {2018},
	pages = {1-12},
	doi = {10.1007/s11263-018-1070-x},
	author = {Hassan Abu Alhaija and Mustikovela, S K and Mescheder, A and Geiger, C and Carsten Rother}
}
@article {AbuAlhaija2018,
	title = {Augmented Reality Meets Computer Vision: Efficient Data Generation for Urban Driving Scenes},
	journal = {International Journal of Computer Vision},
	volume = {126},
	number = {9},
	year = {2018},
	month = {aug},
	pages = {961{\textendash}972},
	abstract = {The success of deep learning in computer vision is based on the availability of large annotated datasets. To lower the need for hand labeled images, virtually rendered 3D worlds have recently gained popularity. Unfortunately, creating realistic 3D content is challenging on its own and requires significant human effort. In this work, we propose an alternative paradigm which combines real and synthetic data for learning semantic instance segmentation and object detection models. Exploiting the fact that not all aspects of the scene are equally important for this task, we propose to augment real-world imagery with virtual objects of the target category. Capturing real-world images at large scale is easy and cheap, and directly provides real background appearances without the need for creating complex 3D models of the environment. We present an efficient procedure to augment these images with virtual objects. In contrast to modeling complete 3D environments, our data augmentation approach requires only a few user interactions in combination with 3D models of the target object category. Leveraging our approach, we introduce a novel dataset of augmented urban driving scenes with 360 degree images that are used as environment maps to create realistic lighting and reflections on rendered objects. We analyze the significance of realistic object placement by comparing manual placement by humans to automatic methods based on semantic scene analysis. This allows us to create composite images which exhibit both realistic background appearance as well as a large number of complex object arrangements. Through an extensive set of experiments, we conclude the right set of parameters to produce augmented data which can maximally enhance the performance of instance segmentation models. Further, we demonstrate the utility of the proposed approach on training standard deep models for semantic instance segmentation and object detection of cars in outdoor driving scenarios. We test the models trained on our augmented data on the KITTI 2015 dataset, which we have annotated with pixel-accurate ground truth, and on the Cityscapes dataset. Our experiments demonstrate that the models trained on augmented imagery generalize better than those trained on fully synthetic data or models trained on limited amounts of annotated real data.},
	keywords = {autonomous driving, Data augmentation, instance segmentation, Object detection, synthetic training data},
	issn = {15731405},
	doi = {10.1007/s11263-018-1070-x},
	url = {http://arxiv.org/abs/1708.01566},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Mescheder, Lars and Geiger, Andreas and Carsten Rother}
}
@conference {Jafari2017,
	title = {Analyzing modular CNN architectures for joint depth prediction and semantic segmentation},
	booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
	year = {2017},
	month = {feb},
	pages = {4620{\textendash}4627},
	abstract = {This paper addresses the task of designing a modular neural network architecture that jointly solves different tasks. As an example we use the tasks of depth estimation and semantic segmentation given a single RGB image. The main focus of this work is to analyze the cross-modality influence between depth and semantic prediction maps on their joint refinement. While most of the previous works solely focus on measuring improvements in accuracy, we propose a way to quantify the cross-modality influence. We show that there is a relationship between final accuracy and cross-modality influence, although not a simple linear one. Hence a larger cross-modality influence does not necessarily translate into an improved accuracy. We find that a beneficial balance between the cross-modality influences can be achieved by network architecture and conjecture that this relationship can be utilized to understand different network design choices. Towards this end we propose a Convolutional Neural Network (CNN) architecture that fuses the state-of-the-art results for depth estimation and semantic labeling. By balancing the cross-modality influences between depth and semantic prediction, we achieve improved results for both tasks using the NYU-Depth v2 benchmark.},
	isbn = {9781509046331},
	issn = {10504729},
	doi = {10.1109/ICRA.2017.7989537},
	url = {http://arxiv.org/abs/1702.08009 http://dx.doi.org/10.1109/ICRA.2017.7989537},
	author = {Omid Hosseini Jafari and Groth, Oliver and Kirillov, Alexander and Yang, Michael Ying and Carsten Rother}
}
@conference {Alhaija2017,
	title = {Augmented reality meets deep learning for car instance segmentation in urban scenes},
	booktitle = {British Machine Vision Conference 2017, BMVC 2017},
	year = {2017},
	abstract = {The success of deep learning in computer vision is based on the availability of large annotated datasets. To lower the need for hand labeled images, virtually rendered 3D worlds have recently gained popularity. Unfortunately, creating realistic 3D content is challenging on its own and requires significant human effort. In this work, we propose an alternative paradigm which combines real and synthetic data for learning semantic instance segmentation models. Exploiting the fact that not all aspects of the scene are equally important for this task, we propose to augment real-world imagery with virtual objects of the target category. Capturing real-world images at large scale is easy and cheap, and directly provides real background appearances without the need for creating complex 3D models of the environment. We present an efficient procedure to augment these images with virtual objects. This allows us to create realistic composite images which exhibit both realistic background appearance as well as a large number of complex object arrangements. In contrast to modeling complete 3D environments, our data augmentation approach requires only a few user interactions in combination with 3D shapes of the target object category. We demonstrate the utility of the proposed approach for training a state-of-the-art high-capacity deep model for semantic instance segmentation. In particular, we consider the task of segmenting car instances on the KITTI dataset which we have annotated with pixel-accurate ground truth. Our experiments demonstrate that models trained on augmented imagery generalize better than those trained on synthetic data or models trained on limited amounts of annotated real data.},
	isbn = {190172560X},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Mescheder, Lars and Geiger, Andreas and Carsten Rother}
}
@conference {Kainmueller2014,
	title = {Active graph matching for automatic joint segmentation and annotation of C. elegans},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {8673 LNCS},
	number = {PART 1},
	year = {2014},
	pages = {81{\textendash}88},
	abstract = {In this work we present a novel technique we term active graph matching, which integrates the popular active shape model into a sparse graph matching problem. This way we are able to combine the benefits of a global, statistical deformation model with the benefits of a local deformation model in form of a second-order random field. We present a new iterative energy minimization technique which achieves empirically good results. This enables us to exceed state-of-the art results for the task of annotating nuclei in 3D microscopic images of C. elegans. Furthermore with the help of the generalized Hough transform we are able to jointly segment and annotate a large set of nuclei in a fully automatic fashion for the first time. {\textcopyright} 2014 Springer International Publishing.},
	isbn = {9783319104034},
	issn = {16113349},
	doi = {10.1007/978-3-319-10404-1_11},
	author = {Kainmueller, Dagmar and Jug, Florian and Carsten Rother and Myers, Gene}
}
@conference {Sindeev2013,
	title = {Alpha-flow for video matting},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {7726 LNCS},
	number = {PART 3},
	year = {2013},
	pages = {438{\textendash}452},
	abstract = {This work addresses the problem of video matting, that is extracting the opacity-layer of a foreground object from a video sequence. We introduce the notion of alpha-flow which corresponds to the flow in the opacity layer. The idea is derived from the process of rotoscoping, where a user-supplied object mask is smoothly interpolated between keyframes while preserving its correspondence with the underlying image. Our key contribution is an algorithm which infers both the opacity masks and the alpha-flow in an efficient and unified manner. We embed our algorithm in an interactive video matting system where the first and last frame of a sequence are given as keyframes, and additional user strokes may be provided in intermediate frames. We show high quality results on various challenging sequences, and give a detailed comparison to competing techniques. {\textcopyright} 2013 Springer-Verlag.},
	isbn = {9783642374302},
	issn = {03029743},
	doi = {10.1007/978-3-642-37431-9_34},
	author = {Sindeev, Mikhail and Konushin, Anton and Carsten Rother}
}
@conference {Kolmogorov2007a,
	title = {Applications of parametric maxflow in computer vision},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2007},
	abstract = {The maximum flow algorithm for minimizing energy functions of binary variables has become a standard tool in computer vision. In many cases, unary costs of the energy depend linearly on parameter λ. In this paper we study vision applications for which it is important to solve the max-flow problem for different λ{\textquoteright}s. An example is a weighting between data and regularization terms in image segmentation or stereo: it is desirable to vary it both during training (to learn λ from ground truth data) and testing (to select best λ using high-knowledge constraints, e.g. user input). We review algorithmic aspects of this parametric maximum flow problem previously unknown in vision, such as the ability to compute all breakpoints of λ and corresponding optimal configurations in finite time. These results allow, in particular, to minimize the ratio of some geometric functionals, such as flux of a vector field over length (or area). Previously, such functionals were tackled with shortest path techniques applicable only in 2D. We give theoretical improvements for "PDE cuts" [5]. We present experimental results for image segmentation, 3D reconstruction, and the cosegmentation problem. {\textcopyright}2007 IEEE.},
	doi = {10.1109/ICCV.2007.4408910},
	author = {Kolmogorov, Vladimir and Boykov, Yuri and Carsten Rother}
}
@conference {Kolmogorov2007b,
	title = {Applications of parametric maxflow in computer vision},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2007},
	abstract = {The maximum flow algorithm for minimizing energy functions of binary variables has become a standard tool in computer vision. In many cases, unary costs of the energy depend linearly on parameter λ. In this paper we study vision applications for which it is important to solve the max-flow problem for different λ{\textquoteright}s. An example is a weighting between data and regularization terms in image segmentation or stereo: it is desirable to vary it both during training (to learn λ from ground truth data) and testing (to select best λ using high-knowledge constraints, e.g. user input). We review algorithmic aspects of this parametric maximum flow problem previously unknown in vision, such as the ability to compute all breakpoints of λ and corresponding optimal configurations in finite time. These results allow, in particular, to minimize the ratio of some geometric functionals, such as flux of a vector field over length (or area). Previously, such functionals were tackled with shortest path techniques applicable only in 2D. We give theoretical improvements for "PDE cuts" [5]. We present experimental results for image segmentation, 3D reconstruction, and the cosegmentation problem. {\textcopyright}2007 IEEE.},
	doi = {10.1109/ICCV.2007.4408910},
	author = {Kolmogorov, Vladimir and Boykov, Yuri and Carsten Rother}
}
