@article{lamberti2024drone,
  author = {Lamberti, Lorenzo and Cereda, Elia and Abbate, Gabriele and Bellone, Lorenzo and Morinigo, Victor Javier Kartsch and Barciś, Michał and Barciś, Agata and Giusti, Alessandro and Conti, Francesco and Palossi, Daniele},
  journal = {IEEE Robotics and Automation Letters},
  title = {A Sim-to-Real Deep Learning-Based Framework for Autonomous Nano-Drone Racing},
  year = {2024},
  volume = {9},
  number = {2},
  pages = {1899-1906},
  keywords = {},
  doi = {10.1109/LRA.2024.3349814},
  issn = {2377-3766},
  month = feb,
  file = {lamberti2024drone.pdf},
  video = {https://www.youtube.com/watch?v=vHTAwUsj-nk}
}

Autonomous drone racing competitions are a proxy to improve unmanned aerial vehicles’ perception, planning, and control skills. The recent emergence of autonomous nano-sized drone racing imposes new challenges, as their ∼\text10 \,\textc\textm form factor heavily restricts the resources available onboard, including memory, computation, and sensors. This letter describes the methodology and technical implementation of the system winning the first autonomous nano-drone racing international competition: the “IMAV 2022 Nanocopter AI Challenge.” We developed a fully onboard deep learning approach for visual navigation trained only on simulation images to achieve this goal. Our approach includes a convolutional neural network for obstacle avoidance, a sim-to-real dataset collection procedure, and a navigation policy that we selected, characterized, and adapted through simulation and actual in-field experiments. Our system ranked 1\textst among six competing teams at the competition. In our best attempt, we scored \text115 \,\textm of traveled distance in the allotted 5-minute flight, never crashing while dodging static and dynamic obstacles. Sharing our knowledge with the research community, we aim to provide a solid groundwork to foster future development in this field.

@article{abbate2023mirror,
  author = {Abbate, Gabriele and Giusti, Alessandro and Randazzo, Luca and Paolillo, Antonio},
  year = {2023},
  month = dec,
  pages = {22881},
  title = {A mirror therapy system using virtual reality and an actuated exoskeleton for the recovery of hand motor impairments: a study of acceptability, usability, and embodiment},
  volume = {13},
  journal = {Scientific Reports},
  doi = {10.1038/s41598-023-49571-7},
  file = {abbate2023mirror.pdf},
  video = {https://www.youtube.com/watch?v=znBTWJ_E1Bc}
}

Hand motor impairments are one of the main causes of disabilities worldwide. Rehabilitation procedures like mirror therapy are given crucial importance. In the traditional setup, the patient moves the healthy hand in front of a mirror; the view of the mirrored motion tricks the brain into thinking that the impaired hand is moving as well, stimulating the recovery of the lost hand functionalities. We propose an innovative mirror therapy system that leverages and couples cutting-edge technologies. Virtual reality recreates an immersive and effective mirroring effect; a soft hand exoskeleton accompanies the virtual visual perception by physically inducing the mirrored motion to the real hand. Three working modes of our system have been tested with 21 healthy users. The system is ranked as acceptable by the system usability scale; it does not provoke adverse events or sickness in the users, according to the simulator sickness questionnaire; the three execution modes are also compared w.r.t. the sense of embodiment, evaluated through another customized questionnaire. The achieved results show the potential of our system as a clinical tool and reveal its social and economic impact.

@article{abbate2023self,
  title = {Self-supervised prediction of the intention to interact with a service robot},
  author = {Abbate, Gabriele and Giusti, Alessandro and Schmuck, Viktor and Celiktutan, Oya and Paolillo, Antonio},
  journal = {Robotics and Autonomous Systems},
  pages = {104568},
  year = {2023},
  publisher = {North-Holland},
  doi = {10.1016/j.robot.2023.104568},
  file = {abbate2023self.pdf},
  video = {https://www.youtube.com/watch?v=mdZDIsr5tcU}
}

A service robot can provide a smoother interaction experience if it has the ability to proactively detect whether a nearby user intends to interact, in order to adapt its behavior e.g. by explicitly showing that it is available to provide a service. In this work, we propose a learning-based approach to predict the probability that a human user will interact with a robot before the interaction actually begins; the approach is self-supervised because after each encounter with a human, the robot can automatically label it depending on whether it resulted in an interaction or not. We explore different classification approaches, using different sets of features considering the pose and the motion of the user. We validate and deploy the approach in three scenarios. The first collects 3442 natural sequences (both interacting and non-interacting) representing employees in an office break area: a real-world, challenging setting, where we consider a coffee machine in place of a service robot. The other two scenarios represent researchers interacting with service robots (200 and 72 sequences, respectively). Results show that, even in challenging real-world settings, our approach can learn without external supervision, and can achieve accurate classification (i.e. AUROC greater than 0.9) of the user’s intention to interact with an advance of more than 3 s before the interaction actually occurs.

@article{paolillo2022towards,
  title = {Towards the integration of a pointing-based human-machine interface in an industrial control system compliant with the iec 61499 standard},
  author = {Paolillo, Antonio and Abbate, Gabriele and Giusti, Alessandro and Traki{\'c}, {\v{S}}ejla and Dzafic, Hilmo and Fritz, Artur and Guzzi, J{\'e}r{\^o}me},
  journal = {Procedia CIRP},
  volume = {107},
  pages = {1077--1082},
  year = {2022},
  publisher = {Elsevier},
  doi = {10.1016/j.procir.2022.05.111},
  file = {paolillo2022towards.pdf}
}

In the context of Industry 4.0, we tackle the problem of how human operators can interact with a cyber-physical system at run-time. We focus on sporadic interactions where operators, normally occupied with other activities, need to communicate to the system a piece of information relative to a specif part of the plant, such as an anomaly. As a concrete instance of this problem, we consider an automated system composed of a robot loading packages on a conveyor belt that transports and sorts them. We argue that gesture-based interaction modalities offer important advantages for the operators in the considered scenario and can be deployed respecting the IEC 61499 standard.

@article{can2021semantic,
  title = {Semantic segmentation on Swiss3DCities: A benchmark study on aerial photogrammetric 3D pointcloud dataset},
  author = {Can, G{\"u}lcan and Mantegazza, Dario and Abbate, Gabriele and Chappuis, S{\'e}bastien and Giusti, Alessandro},
  journal = {Pattern Recognition Letters},
  volume = {150},
  pages = {108--114},
  year = {2021},
  publisher = {Elsevier},
  doi = {10.1016/j.patrec.2021.06.004},
  file = {can2021semantic.pdf},
  info = {https://github.com/idsia-robotics/RandLA-Net-pytorch}
}

We introduce a new outdoor urban 3D pointcloud dataset, covering a total area of 2.7 km 2, sampled from three Swiss cities with different characteristics. The dataset is manually annotated for semantic segmentation with per-point labels, and is built using photogrammetry from images acquired by multirotors equipped with high-resolution cameras. In contrast to datasets acquired with ground LiDAR sensors, the resulting point clouds are uniformly dense and complete, and are useful to disparate applications, including autonomous driving, gaming and smart city planning. As a benchmark, we report quantitative results of PointNet++, an established point-based deep 3D semantic segmentation model; on this model, we additionally study the impact of using different cities for model generalization.

@inproceedings{abbate2022pointit,
  title = {PointIt: A ROS toolkit for interacting with co-located robots using pointing gestures},
  author = {Abbate, Gabriele and Giusti, Alessandro and Paolillo, Antonio and Gromov, Boris and Gambardella, Luca and Rizzoli, Andrea Emilio and Guzzi, J{\'e}r{\^o}me},
  booktitle = {2022 17th ACM/IEEE International Conference on Human-Robot Interaction (HRI)},
  pages = {608--612},
  year = {2022},
  organization = {IEEE},
  doi = {10.1109/HRI53351.2022.9889486},
  file = {abbate2022pointit.pdf},
  info = {https://github.com/Gabry993/pointing-user-interface-hri}
}

We introduce PointIt, a toolkit for the Robot Operating System (ROS2) to build human-robot interfaces based on pointing gestures sensed by a wrist-worn Inertial Measurement Unit, such as a smartwatch. We release the software as open-source with MIT license; docker images and exhaustive instructions simplify its usage in simulated and real-world deployments.

@inproceedings{guzzi2022interacting,
  title = {Interacting with a conveyor belt in virtual reality using pointing gestures},
  author = {Guzzi, J{\'e}r{\^o}me and Abbate, Gabriele and Paolillo, Antonio and Giusti, Alessandro},
  booktitle = {2022 17th ACM/IEEE International Conference on Human-Robot Interaction (HRI)},
  pages = {1194--1195},
  year = {2022},
  organization = {IEEE},
  doi = {10.1109/HRI53351.2022.9889380},
  file = {guzzi2022interacting.pdf},
  video = {https://idsia-robotics.github.io/pointing/files/videos/guzzi2022demo.mp4}
}

We present an interactive demonstration where users are immersed in a virtual reality simulation of a logistic automation system. Using pointing gestures sensed by wrist-worn inertial measurement unit, users select defective packages transported on conveyor belts. The demonstration allows users to experience a novel way to interact with automation systems, and shows an effective application of virtual reality for human-robot interaction studies.

@inproceedings{abbate2022selecting,
  title = {Selecting Objects on Conveyor Belts Using Pointing Gestures Sensed by a Wrist-worn Inertial Measurement Unit},
  author = {Abbate, Gabriele and Giusti, Alessandro and Paolillo, Antonio and Gambardella, Luca Maria and Rizzoli, Andrea Emilio and Guzzi, J{\'e}r{\^o}me},
  booktitle = {2022 IEEE 18th International Conference on Automation Science and Engineering (CASE)},
  pages = {633--640},
  year = {2022},
  organization = {IEEE},
  doi = {10.1109/CASE49997.2022.9926448},
  file = {abbate2022selecting.pdf},
  video = {https://www.youtube.com/watch?v=fEnPEXPlVIM}
}

We introduce an intuitive pointing-based interface to select objects moving on a system of conveyor belts. The interface has minimal sensing requirements, as the operator only needs to wear an Inertial Measurement Unit on the wrist (e.g., a smartwatch). LED strips provide the required visual feedback to precisely point to the objects and select them. We test the proposed approach in three environments of different complexity. Experiments compare our approach with a graphical interface where the user clicks on packages with a mouse; quantitative results show that our interface compares favorably, especially in difficult scenarios involving many packages moving fast.

@inproceedings{abbate2021pointing,
  title = {Pointing at moving robots: Detecting events from wrist IMU data},
  author = {Abbate, Gabriele and Gromov, Boris and Gambardella, Luca M and Giusti, Alessandro},
  booktitle = {2021 IEEE International Conference on Robotics and Automation (ICRA)},
  pages = {3604--3611},
  year = {2021},
  organization = {IEEE},
  doi = {10.1109/ICRA.2019.8794399},
  file = {abbate2021pointing.pdf},
  video = {https://www.youtube.com/embed/-Eroidxx9sY?si=MuZBG5ogLqvOBzMR}
}

We present a system for interaction between co-located humans and mobile robots, which uses pointing gestures sensed by a wrist-mounted IMU. The operator begins by pointing, for a short time, at a moving robot. The system thus simultaneously determines: that the operator wants to interact; the robot they want to interact with; and the relative pose among the two. Then, the system can reconstruct pointed locations in the robot’s own reference frame, and provide real-time feedback about them so that the user can adapt to misalignments. We discuss the challenges to be solved to implement such a system and propose practical solutions, including variants for fast flying robots and slow ground robots. We report different experiments with real robots and untrained users, validating the individual components and the system as a whole.

@inproceedings{gromov2019proximity,
  title = {Proximity human-robot interaction using pointing gestures and a wrist-mounted IMU},
  author = {Gromov, Boris and Abbate, Gabriele and Gambardella, Luca M and Giusti, Alessandro},
  booktitle = {2019 International Conference on Robotics and Automation (ICRA)},
  pages = {8084--8091},
  year = {2019},
  organization = {IEEE},
  doi = {10.1109/ICRA.2019.8794399},
  file = {gromov2019proximity.pdf}
}

We present a system for interaction between co-located humans and mobile robots, which uses pointing gestures sensed by a wrist-mounted IMU. The operator begins by pointing, for a short time, at a moving robot. The system thus simultaneously determines: that the operator wants to interact; the robot they want to interact with; and the relative pose among the two. Then, the system can reconstruct pointed locations in the robot’s own reference frame, and provide real-time feedback about them so that the user can adapt to misalignments. We discuss the challenges to be solved to implement such a system and propose practical solutions, including variants for fast flying robots and slow ground robots. We report different experiments with real robots and untrained users, validating the individual components and the system as a whole.

@inproceedings{gromov2019video,
  title = {Video: Pointing gestures for proximity interaction},
  author = {Gromov, Boris and Guzzi, J{\'e}r{\^o}me and Abbate, Gabriele and Gambardella, Luca and Giusti, Alessandro},
  booktitle = {2019 14th ACM/IEEE International Conference on Human-Robot Interaction (HRI)},
  pages = {370--370},
  year = {2019},
  organization = {IEEE},
  doi = {10.1109/HRI.2019.8673020},
  file = {gromov2019video.pdf},
  video = {https://www.youtube.com/watch?v=yafy-HZMk_U},
  award = {hri2019_award.jpg}
}

We propose a system to control robots in the users proximity with pointing gestures-a natural device that people use all the time to communicate with each other. Our system has two requirements: first, the robot must be able to reconstruct its own motion, e.g. by means of visual odometry; second, the user must wear a wristband or smartwatch with an inertial measurement unit. Crucially, the robot does not need to perceive the user in any way. The resulting system is widely applicable, robust, and intuitive to use.