Skip to content

Revert "Ffmpeg stream update" #97

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nickgaray merged 1 commit into from
Oct 7, 2026
Merged

Revert "Ffmpeg stream update" #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 0 additions & 31 deletions sensors/video/sensorhub-driver-ffmpeg/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,37 +63,6 @@ When added to an OpenSensorHub node, the driver has the following configuration
- **Register Devices:**
Register external A/V device libraries. Enable if input is not from a network stream or file (e.g. Direct Show, V4L).
Disable this option if the driver fails to start and produces the error `java.lang.UnsatisfiedLinkError: no jniavdevice in java.library.path`.
- **Stream Filter:**
Select sub-streams to be processed and outputted. If left unselected, the default is one video and one audio output.
- **Max Count:**
Select the maximum number of streams for each type (video, audio, binary data) to process and output.
The default is one video, one audio, and zero binary data streams.
- **Stream Index:**
Select streams based on their index. Specify indices using comma-separated values.
Select a range of indices using a hyphen-separated value.
For example, "0, 2-4, 6" will select streams 0, 2, 3, 4, and 6.
- For the stream filter selection UI to work, add the following to AdminUIConfig customForms
in config.json:
```json
{
"objClass": "org.sensorhub.ui.AdminUIConfig",
"widgetSet": "org.sensorhub.ui.SensorHubWidgetSet",
"bundleRepoUrls": [],
"customPanels": [],
"customForms": [
{
"objClass": "org.sensorhub.ui.CustomUIConfig",
"configClass": "org.sensorhub.impl.sensor.ffmpeg.config.Connection",
"uiClass": "com.georobotix.ui.ffmpeg.forms.FFmpegStreamConfigForm"
}
],
"enableLandingPage": false,
"id": "5cb05c9c-9123-4fa1-8731-ffaa51489678",
"autoStart": true,
"moduleClass": "org.sensorhub.ui.AdminUIModule",
"name": "Admin UI"
}
```

- **Position:**
- **Location:** (Optional)
Expand Down
2 changes: 1 addition & 1 deletion sensors/video/sensorhub-driver-ffmpeg/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
description = 'FFMPEG Video Driver'
ext.details = "Driver for connecting to ffmpeg capable video streams"
version = '1.2.0'
version = '1.1.0'

// To build in entire java cv platform of projects use
// implementation('org.bytedeco:javacv-platform:1.5.2')
Expand Down
267 changes: 38 additions & 229 deletions .../sensorhub-driver-ffmpeg/src/main/java/org/sensorhub/impl/sensor/ffmpeg/FFMPEGSensor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,15 @@
package org.sensorhub.impl.sensor.ffmpeg;

import org.sensorhub.api.common.SensorHubException;
import org.sensorhub.impl.module.AbstractModule;
import org.sensorhub.impl.sensor.AbstractSensorModule;
import org.sensorhub.impl.sensor.ffmpeg.config.FFMPEGConfig;
import org.sensorhub.impl.sensor.ffmpeg.config.streamfilter.MaxCountFilter;
import org.sensorhub.impl.sensor.ffmpeg.outputs.AudioOutput;
import org.sensorhub.impl.sensor.ffmpeg.outputs.DataOutput;
import org.sensorhub.impl.sensor.ffmpeg.outputs.VideoOutput;
import org.sensorhub.mpegts.MpegTsProcessor;
import org.sensorhub.mpegts.StreamContext;
import org.vast.swe.SWEConstants;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.net.UnknownHostException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;

Expand All @@ -44,38 +39,15 @@ public class FFMPEGSensor extends AbstractSensorModule<FFMPEGConfig> {
protected ScheduledExecutorService executor;

/**
* Sensor output for the video frames of the first video stream of the source.
* The outputs for any further video streams are in {@link FFMPEGSensor#videoOutputs}.
* Sensor output for the video frames.
*/
protected VideoOutput<FFMPEGSensor> videoOutput;

/**
* Sensor output for the audio data of the first audio stream of the source.
* The outputs for any further audio streams are in {@link FFMPEGSensor#audioOutputs}.
* Sensor output for the audio data.
*/
protected AudioOutput<FFMPEGSensor> audioOutput;

/**
* Sensor output for the raw bytes of the first binary data stream of the source.
* The outputs for any further data streams are in {@link FFMPEGSensor#dataOutputs}.
*/
protected DataOutput<FFMPEGSensor> dataOutput;

/**
* All video outputs, ordered by stream ID. The first entry is also held in {@link FFMPEGSensor#videoOutput}.
*/
protected final List<VideoOutput<FFMPEGSensor>> videoOutputs = new ArrayList<>();

/**
* All audio outputs, ordered by stream ID. The first entry is also held in {@link FFMPEGSensor#audioOutput}.
*/
protected final List<AudioOutput<FFMPEGSensor>> audioOutputs = new ArrayList<>();

/**
* All binary data outputs, ordered by stream ID. The first entry is also held in {@link FFMPEGSensor#dataOutput}.
*/
protected final List<DataOutput<FFMPEGSensor>> dataOutputs = new ArrayList<>();

@Override
protected void doInit() throws SensorHubException {
super.doInit();
Expand All @@ -88,11 +60,6 @@ protected void doInit() throws SensorHubException {
if (config.connection.fps < 0)
throw new SensorHubException("FPS must be a positive value");

if (config.connection.streamFilter == null) {
logger.warn("No stream filter specified, using default");
config.connection.streamFilter = new MaxCountFilter();
}

// Every time we do init we have to tear down the mpegTsProcessor,
// just in case they changed some setting that might cause the video output to be different.
if (mpegTsProcessor != null) {
Expand All @@ -107,14 +74,10 @@ protected void doInit() throws SensorHubException {
}
}

// We also have to clear out the outputs since their settings may have changed
// We also have to clear out the video output since its settings may have changed
// (based on having a new input video, for example).
videoOutput = null;
audioOutput = null;
dataOutput = null;
videoOutputs.clear();
audioOutputs.clear();
dataOutputs.clear();

// We need the background thread here since we start reading the video data immediately to determine the video size.
setupExecutor();
Expand Down Expand Up @@ -179,9 +142,10 @@ protected void setupExecutor() {
logger.debug("Already had an executor.");
}

videoOutputs.forEach(output -> output.setExecutor(executor));
audioOutputs.forEach(output -> output.setExecutor(executor));
dataOutputs.forEach(output -> output.setExecutor(executor));
if (videoOutput != null)
videoOutput.setExecutor(executor);
if (audioOutput != null)
audioOutput.setExecutor(executor);
}

/**
Expand All @@ -201,68 +165,25 @@ protected void shutdownExecutor() {
* The caller must be careful not to call this if the video output has already been created and added to the sensor.
*/
protected void createVideoOutput(int[] videoDims, String codecName) {
int index = videoOutputs.size();
String name = outputName("video", index);

var output = new VideoOutput<FFMPEGSensor>(this, videoDims, codecName, name,
outputLabel("Video", index), "Video stream using ffmpeg library");

videoOutput = new VideoOutput<>(this, videoDims, codecName);
if (executor != null) {
output.setExecutor(executor);
videoOutput.setExecutor(executor);
}
addOutput(output, false);
output.doInit();
videoOutputs.add(output);

// Keep the field pointing at the primary video output
if (videoOutput == null)
videoOutput = output;
addOutput(videoOutput, false);
videoOutput.doInit();
}

/**
* Create and initialize the audio output.
* The caller must be careful not to call this if the audio output has already been created and added to the sensor.
*/
protected void createAudioOutput(int sampleRate, String codecName) {
int index = audioOutputs.size();
String name = outputName("audio", index);

var output = new AudioOutput<FFMPEGSensor>(this, sampleRate, codecName, name,
outputLabel("Audio", index), "Audio stream using ffmpeg library");

if (executor != null) {
output.setExecutor(executor);
}
addOutput(output, false);
output.doInit();
audioOutputs.add(output);

// Keep the field pointing at the primary audio output
if (audioOutput == null)
audioOutput = output;
}

/**
* Create and initialize the binary data output.
* The caller must be careful not to call this if the data output has already been created and added to the sensor.
*/
protected void createDataOutput() {
int index = dataOutputs.size();
String name = outputName("data", index);

var output = new DataOutput<FFMPEGSensor>(this, name,
outputLabel("Data", index), "Raw data stream using ffmpeg library");

audioOutput = new AudioOutput<>(this, sampleRate, codecName);
if (executor != null) {
output.setExecutor(executor);
audioOutput.setExecutor(executor);
}
addOutput(output, false);
output.doInit();
dataOutputs.add(output);

// Keep the field pointing at the primary data output
if (dataOutput == null)
dataOutput = output;
addOutput(audioOutput, false);
audioOutput.doInit();
}

/**
Expand Down Expand Up @@ -300,145 +221,33 @@ protected boolean openStream() {

// Initialize the MPEG transport stream processor from the source named in the configuration.
if (mpegTsProcessor.openStream()) {
initializeOutputs();
logger.info("MPEG TS stream for {} opened.", getUniqueIdentifier());
return true;
}

return false;
}

/**
* Called after the stream is opened.
* Initialize outputs based on the stream contents.
* <p>
* An output is created for every video, audio, and binary data stream the source carries,
* filtered by {@link org.sensorhub.impl.sensor.ffmpeg.config.Connection#streamFilter}, and each output is
* registered as the listener for its own stream.
* <p>
* Outputs are reused across stop/start cycles: on restart the stream contexts are rebuilt by the stream
* processor, so the listeners have to be re-registered, but the outputs themselves stay valid.
* <br>Override this method for custom output handling. (Custom data stream output, etc.)
*/
protected void initializeOutputs() {
int videoCount = 0;
int audioCount = 0;
int dataCount = 0;
var streamFilter = config.connection.streamFilter;

for (StreamContext streamContext : streamFilter.getFilteredStreams(mpegTsProcessor.getStreamCollection())) {
switch (streamContext.getStreamType()) {
case VIDEO -> {
attachVideoOutput(streamContext, videoCount++);
}
case AUDIO -> {
attachAudioOutput(streamContext, audioCount++);
// If there is a video content in the stream
if (mpegTsProcessor.hasVideoStream()) {
// In case we were waiting until we got video data to make the video frame output,
// we go ahead and do that now.
if (videoOutput == null) {
createVideoOutput(mpegTsProcessor.getVideoStreamFrameDimensions(), mpegTsProcessor.getVideoCodecName());
}
case DATA -> {
attachDataOutput(streamContext, dataCount++);
}
default -> logger.debug("Ignoring stream {} of type {}",
streamContext.getStreamId(), streamContext.getStreamType());
// Set video stream packet listener to video output
mpegTsProcessor.setVideoDataBufferListener(videoOutput);
}
}

logger.info("Initialized {} video, {} audio, and {} binary data output(s) for {}",
videoCount, audioCount, dataCount, getUniqueIdentifier());

if (videoCount == 0 && audioCount == 0 && dataCount == 0)
reportStatus("No video, audio, or binary data streams published from " + config.connection.connectionString);
}

/**
* @return All video outputs of this driver, ordered by stream ID.
*/
public List<VideoOutput<FFMPEGSensor>> getVideoOutputs() {
return Collections.unmodifiableList(videoOutputs);
}

/**
* @return All audio outputs of this driver, ordered by stream ID.
*/
public List<AudioOutput<FFMPEGSensor>> getAudioOutputs() {
return Collections.unmodifiableList(audioOutputs);
}

/**
* @return All binary data outputs of this driver, ordered by stream ID.
*/
public List<DataOutput<FFMPEGSensor>> getDataOutputs() {
return Collections.unmodifiableList(dataOutputs);
}

/**
* Creates the video output for the given stream if it does not exist yet, then registers it as the
* listener for that stream.
*
* @param streamContext The video stream to publish.
* @param index Zero-based ordinal of this stream among the video streams of the source.
*/
protected void attachVideoOutput(StreamContext streamContext, int index) {
if (index >= videoOutputs.size()) {
createVideoOutput(streamContext.getFrameDimensions(), streamContext.getCodecName());

logger.info("Created video output '{}' for stream {} ({}, {}x{})", videoOutputs.get(index).getName(),
streamContext.getStreamId(), streamContext.getCodecName(),
streamContext.getFrameWidth(), streamContext.getFrameHeight());
}

streamContext.setDataBufferListener(videoOutputs.get(index));
}

/**
* Creates the audio output for the given stream if it does not exist yet, then registers it as the
* listener for that stream.
*
* @param streamContext The audio stream to publish.
* @param index Zero-based ordinal of this stream among the audio streams of the source.
*/
protected void attachAudioOutput(StreamContext streamContext, int index) {
if (index >= audioOutputs.size()) {
createAudioOutput(streamContext.getSampleRate(), streamContext.getCodecName());

logger.info("Created audio output '{}' for stream {} ({}, {} Hz)", audioOutputs.get(index).getName(),
streamContext.getStreamId(), streamContext.getCodecName(), streamContext.getSampleRate());
}

streamContext.setDataBufferListener(audioOutputs.get(index));
}

/**
* Creates the binary data output for the given stream if it does not exist yet, then registers it as the
* listener for that stream.
*
* @param streamContext The data stream to publish.
* @param index Zero-based ordinal of this stream among the data streams of the source.
*/
protected void attachDataOutput(StreamContext streamContext, int index) {
if (index >= dataOutputs.size()) {
createDataOutput();
// If there is an audio content in the stream
if (mpegTsProcessor.hasAudioStream()) {
// In case we were waiting until we got audio data to make the audio output,
// we go ahead and do that now.
if (audioOutput == null) {
createAudioOutput(mpegTsProcessor.getAudioSampleRate(), mpegTsProcessor.getAudioCodecName());
}
// Set audio stream packet listener to audio output
mpegTsProcessor.setAudioDataBufferListener(audioOutput);
}

logger.info("Created binary data output '{}' for stream {} (codec {}, tag '{}', handler '{}')",
dataOutputs.get(index).getName(), streamContext.getStreamId(), streamContext.getCodecName(),
streamContext.getCodecTagString(), streamContext.getHandlerName());
logger.info("MPEG TS stream for {} opened.", getUniqueIdentifier());
return true;
}

streamContext.setDataBufferListener(dataOutputs.get(index));
}

/**
* Builds the output name for a stream, leaving the first stream of each kind unsuffixed so that it
* matches the name a single-stream source would have used.
*/
protected static String outputName(String baseName, int index) {
return index == 0 ? baseName : baseName + (index + 1);
}

/**
* Builds the output label for a stream, leaving the first stream of each kind unsuffixed.
*/
protected static String outputLabel(String baseLabel, int index) {
return index == 0 ? baseLabel : baseLabel + " " + (index + 1);
return false;
}

/**
Expand Down
Loading
Loading