GStreamer is hanging unexpectedly during dynamic change of elements - rust

Question

We are using GStreamer to read the video stream off of network cameras and we observed a weird behaviour: our program would hang unexpectedly, without any kind of notice.
We, then, narrowed it down to the fact that our client had a very, very bad internet connection and we were able to reproduce the problem with just GStreamer and a tool to simulate bad networks.
I am attaching below both our tool to simulate bad networks and the example we've written to reproduce this behaviour.
Our tool to simulate bad networks uses tc to introduce a delay, packet loss and packet corruption.
Our example boils down to using a (subclassed) bin that can intercept EoS messages posted inside it, and a pipeline that reads a RTSP feed. The bin is then dinamically added and removed from the pipeline.
Are we doing something wrong? Could this be a bug in GStreamer itself?
crapnet.sh
sudo tc qdisc add dev wlp2s0 root netem \
delay 500ms 480ms distribution normal \
loss 30% 25% \
corrupt 2%
src/custom_bin.rs
mod implementation {
use {glib::subclass::prelude::*, gstreamer::subclass::prelude::*};
use {
glib::{glib_object_impl, glib_object_subclass, subclass::simple::ClassStruct},
gstreamer::{subclass::ElementInstanceStruct, Bin, Message, MessageView},
std::sync::{mpsc, Mutex},
};
pub struct CustomBin {
pub(super) eos_guard: Mutex<Option<mpsc::SyncSender<()>>>,
}
impl ObjectImpl for CustomBin {
glib_object_impl!();
}
impl ElementImpl for CustomBin {}
impl BinImpl for CustomBin {
fn handle_message(&self, bin: &Bin, message: Message) {
let mut eos_guard = self.eos_guard.lock().unwrap();
if let MessageView::Eos(_) = message.view() {
if let Some(eos_guard) = eos_guard.take() {
return eos_guard.send(()).unwrap_or(());
}
}
self.parent_handle_message(bin, message)
}
}
impl ObjectSubclass for CustomBin {
const NAME: &'static str = "GstCustomBin";
type ParentType = Bin;
type Instance = ElementInstanceStruct<Self>;
type Class = ClassStruct<Self>;
glib_object_subclass!();
fn new() -> Self {
Self {
eos_guard: Mutex::new(None),
}
}
}
}
use {
glib::{prelude::*, subclass::prelude::*, translate::*},
gstreamer::prelude::*,
};
use {
glib::{glib_bool_error, glib_wrapper, subclass::simple::ClassStruct, BoolError, Object},
gstreamer::{
event, subclass::ElementInstanceStruct, Element, GhostPad, PadDirection, State, StateChangeError,
StateChangeSuccess,
},
std::{sync::mpsc, time::Duration},
};
glib_wrapper! {
/// A subclass of `gstreamer::Bin` that has customized behaviour for intercepting end-of-stream messages. This is necessary so that we can remove a branch from a `tee` and not bring down the entire pipeline.
pub struct CustomBin(
Object<
ElementInstanceStruct<implementation::CustomBin>,
ClassStruct<implementation::CustomBin>,
CustomBinClass
>
) #extends gstreamer::Bin, gstreamer::Element, gstreamer::Object;
match fn {
get_type => || implementation::CustomBin::get_type().to_glib(),
}
}
unsafe impl Send for CustomBin {}
unsafe impl Sync for CustomBin {}
impl CustomBin {
/// Instantiates the structure.
pub fn new(name: Option<&str>) -> Self {
Object::new(Self::static_type(), &[("name", &name)])
.expect("Failed to instantiate `CustomBin` as an `Object`")
.downcast()
.expect("Failed to downcast `Object` to `CustomBin`")
}
/// Adds a ghost sink pad for the target element. It is assumed that the element belongs to the bin.
pub fn add_ghost_sink_pad(&self, element: &Element) -> Result<(), BoolError> {
let target_pad = element
.get_static_pad("sink")
.ok_or_else(|| glib_bool_error!("Failed to get sink pad from [{}]", element.get_name()))?;
let ghost_pad = GhostPad::new(Some("sink"), PadDirection::Sink);
ghost_pad.set_target(Some(&target_pad))?;
self.add_pad(&ghost_pad)?;
Ok(())
}
/// Installs an end-of-stream message guard, which will drop the end-of-stream message and then signal it was dropped.
pub fn install_eos_guard(&self) -> mpsc::Receiver<()> {
let super_self = implementation::CustomBin::from_instance(self);
let mut eos_guard = super_self.eos_guard.lock().unwrap();
let (sender, receiver) = mpsc::sync_channel(1);
eos_guard.replace(sender);
receiver
}
/// Sends an end-of-stream event to this bin. It will become an end-of-stream message once it reaches the sink.
pub fn send_eos_event(&self) {
let super_self = implementation::CustomBin::from_instance(self);
let mut eos_guard = super_self.eos_guard.lock().unwrap();
if !self.send_event(event::Eos::new()) {
if let Some(eos_guard) = eos_guard.take() {
eos_guard.send(()).unwrap_or(());
}
}
}
/// Flushes the data in the bin by sending an EoS event and then intercepting the resulting EoS message.
pub fn flush(&self) -> Result<(), BoolError> {
let eos_guard = self.install_eos_guard();
self.send_eos_event();
eos_guard
.recv_timeout(Duration::from_secs(5))
.map_err(|error| glib_bool_error!("Timed out waiting for the EoS guard during flush (error [{:?}])", error))
}
pub fn set_null_state(&self) -> Result<StateChangeSuccess, StateChangeError> {
self.set_state(State::Null)
}
}
src/main.rs
mod custom_bin;
use custom_bin::CustomBin;
use gstreamer::prelude::*;
use {
glib::glib_bool_error,
gstreamer::{ElementFactory, PadProbeReturn, PadProbeType, Pipeline, State},
std::{error::Error, sync::mpsc, thread, time::Duration},
};
fn rtsp_location() -> String {
String::from("rtspt://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mov")
}
fn video_location(index: usize) -> String {
format!("/home/valmirpretto/videos/video_{}.mp4", index)
}
fn instantiate_mp4_writer(location: &str) -> Result<CustomBin, Box<dyn Error>> {
let queue = ElementFactory::make("queue", None)?;
let h264parse = ElementFactory::make("h264parse", None)?;
let mp4mux = ElementFactory::make("mp4mux", None)?;
let filesink = ElementFactory::make("filesink", None)?;
filesink.set_property("location", &location)?;
let bin = CustomBin::new(None);
bin.add(&queue)?;
bin.add(&h264parse)?;
bin.add(&mp4mux)?;
bin.add(&filesink)?;
queue.link(&h264parse)?;
h264parse.link(&mp4mux)?;
mp4mux.link(&filesink)?;
bin.add_ghost_sink_pad(&queue)?;
Ok(bin)
}
fn main() -> Result<(), Box<dyn Error>> {
gstreamer::init()?;
let rtspsrc = ElementFactory::make("rtspsrc", None)?;
let rtph264depay = ElementFactory::make("rtph264depay", None)?;
let tee = ElementFactory::make("tee", None)?;
rtspsrc.set_property("location", &rtsp_location())?;
tee.set_property("allow-not-linked", &true)?;
let pipeline = Pipeline::new(None);
pipeline.add(&rtspsrc)?;
pipeline.add(&rtph264depay)?;
pipeline.add(&tee)?;
rtspsrc.connect_pad_added({
let rtph264depay = rtph264depay.downgrade();
move |_, src_pad| {
if let Some(rtph264depay) = rtph264depay.upgrade() {
let sink_pad = rtph264depay
.get_static_pad("sink")
.expect("Element rtph264depay without sink pad");
if src_pad.can_link(&sink_pad) {
src_pad
.link(&sink_pad)
.expect("Failed to link after we checked it could");
}
}
}
});
rtph264depay.link(&tee)?;
pipeline.set_state(State::Playing)?;
(0..).try_for_each::<_, Result<(), Box<dyn Error>>>(|index| {
println!("Iteration [{}]", index);
// Add bin to the pipeline
let bin = instantiate_mp4_writer(&video_location(index))?;
pipeline.add(&bin)?;
bin.sync_state_with_parent()?;
tee.link(&bin)?;
thread::sleep(Duration::from_secs(5));
// Remove bin from the pipeline
let bin_sink_pad = bin
.get_static_pad("sink")
.ok_or_else(|| glib_bool_error!("Bin [{}] did not have a sink pad", bin.get_name()))?;
let tee_src_pad = bin_sink_pad
.get_peer()
.ok_or_else(|| glib_bool_error!("Bin [{}] sink pad did not have a peer", bin.get_name()))?;
let (signal_sender, signal_receiver) = mpsc::sync_channel(1);
tee_src_pad.add_probe(PadProbeType::IDLE, {
let bin = bin.downgrade();
let pipeline = pipeline.downgrade();
let tee = tee.downgrade();
move |tee_src_pad, _| {
if let Some(bin) = bin.upgrade() {
if let Some(pipeline) = pipeline.upgrade() {
bin.flush().expect(&format!("Could not flush bin [{}]", bin.get_name()));
pipeline
.remove(&bin)
.expect(&format!("Could not remove bin [{}] from pipeline", bin.get_name()));
bin.set_null_state()
.expect(&format!("Could not set null state of bin [{}]", bin.get_name()));
signal_sender
.send(())
.expect(&format!("Could not signal that bin [{}] is done", bin.get_name()));
}
}
if let Some(tee) = tee.upgrade() {
tee.release_request_pad(tee_src_pad);
}
PadProbeReturn::Remove
}
});
signal_receiver
.recv_timeout(Duration::from_secs(5))
.map_err(|error| glib_bool_error!("Timed out waiting for EoS (error [{:?}])", error))?;
Ok(())
})?;
Ok(())
}