#include <perception_server.hpp>

Inheritance diagram for fp_perception::PerceptionServer:

Collaboration diagram for fp_perception::PerceptionServer:

Classes
struct	DriverSpec

Public Types
using	Audio = fp_perception_msgs::msg::PerceptionAudio

using	Speech = fp_perception_msgs::srv::PerceptionSpeech

using	Transcribe = fp_perception_msgs::srv::PerceptionTranscribe

using	Sentiment = fp_perception_msgs::srv::PerceptionSentiment

using	ImageAnalysis = fp_perception_msgs::srv::PerceptionImageAnalysis

Public Member Functions
	PerceptionServer (const rclcpp::NodeOptions &options=rclcpp::NodeOptions())
	Construct a new Perception Server object.

	~PerceptionServer ()

void	initialize ()

Protected Member Functions
void	schedule_deferred_initialization ()

void	start_background_tasks ()

void	stop_background_tasks ()

void	configure_driver_selection ()

void	configure_interfaces ()

template<typename DriverType >
std::shared_ptr< DriverType >	load_driver_if_enabled (const bool enabled, const DriverSpec &spec, const DriverManager &driver_manager)

void	load_drivers (const DriverManager &driver_manager)

void	initialize_pipelines ()

void	run_tests ()

virtual void	publishAudio ()

void	publishVideo ()

void	audioCallback (const Audio &msg)
	Set the latest audio data to the driver from topic subscription. This function is called when new audio data is received from the subscribed topic.

transcription_request	make_transcription_request (const Transcribe::Request &request, std_msgs::msg::Header &header)

void	write_transcription_response (const transcription_result &result, Transcribe::Response &response)

sentiment_request	make_sentiment_request (const Sentiment::Request &request)

void	write_sentiment_response (const sentiment_result &result, Sentiment::Response &response)

image_analysis_request	make_image_analysis_request (const ImageAnalysis::Request &request)

void	write_image_analysis_response (const image_analysis_result &result, ImageAnalysis::Response &response)

void	transcribe_callback (const std::shared_ptr< Transcribe::Request > request, std::shared_ptr< Transcribe::Response > response)
	Service callback for transcription requests.

void	speech_callback (const std::shared_ptr< Speech::Request > request, std::shared_ptr< Speech::Response > response)
	Service callback for the speech service.

void	sentiment_callback (const std::shared_ptr< Sentiment::Request > request, std::shared_ptr< Sentiment::Response > response)
	Callback function for the sentiment analysis service.

fp_perception::audio_data	read_latest_public_audio (int duration_seconds)

fp_perception::audio_data	read_public_audio (const audio_buffer_request &request)

void	image_analysis_callback (const std::shared_ptr< ImageAnalysis::Request > request, std::shared_ptr< ImageAnalysis::Response > response)

Protected Attributes
bool	run_tests_

bool	use_ros_vision_driver_

bool	use_non_ros_vision_driver_

bool	use_microphone_driver_

bool	use_speaker_driver_

bool	use_transcription_driver_

bool	use_sentiment_driver_

bool	use_speech_driver_

bool	use_image_analysis_driver_

AudioBuffer	public_audio_buffer_

std::mutex	transcription_driver_mutex_

std::mutex	sentiment_driver_mutex_

bool	audio_input_publish_

std::string	audio_input_topic_

std::string	audio_input_frame_id_

int	audio_input_frequency_

int	audio_input_retention_window_

int	default_audio_request_window_

std::string	audio_output_topic_

bool	audio_output_subscribe_

bool	vision_input_publish_

std::string	vision_input_topic_

std::string	vision_input_frame_id_

int	vision_input_frequency_

bool	transcription_enabled_

std::string	transcription_service_

bool	speech_enabled_

std::string	speech_service_name_

bool	sentiment_enabled_

std::string	sentiment_service_name_

bool	image_analysis_enabled_

std::string	image_analysis_service_name_

pluginlib::ClassLoader< fp_perception::DriverBase >	driver_loader_

std::shared_ptr< fp_perception::VisionSourceDriver >	ros_vision_driver_

std::shared_ptr< fp_perception::VisionSourceDriver >	non_ros_vision_driver_

std::shared_ptr< fp_perception::AudioSourceDriver >	microphone_driver_

std::shared_ptr< fp_perception::AudioSinkDriver >	speaker_driver_

std::shared_ptr< fp_perception::TranscriptionDriver >	transcription_driver_

std::shared_ptr< fp_perception::SentimentAnalysisDriver >	sentiment_driver_

std::shared_ptr< fp_perception::SpeechSynthesisDriver >	speech_driver_

std::shared_ptr< fp_perception::ImageAnalysisDriver >	image_analysis_driver_

std::unique_ptr< TranscriptionPipeline >	transcription_pipeline_

std::unique_ptr< SpeechPipeline >	speech_pipeline_

std::unique_ptr< SentimentPipeline >	sentiment_pipeline_

std::unique_ptr< ImageAnalysisPipeline >	image_analysis_pipeline_

rclcpp::Publisher< Audio >::SharedPtr	audio_publisher_

rclcpp::Subscription< Audio >::SharedPtr	audio_subscriber_

rclcpp::Service< Transcribe >::SharedPtr	transcription_

rclcpp::Service< ImageAnalysis >::SharedPtr	image_analysis_service_

rclcpp::Service< Speech >::SharedPtr	speech_service_

rclcpp::Service< Sentiment >::SharedPtr	sentiment_service_

rclcpp::Publisher< sensor_msgs::msg::Image >::SharedPtr	image_publisher_

rclcpp::TimerBase::SharedPtr	deferred_initialize_timer_

std::mutex	initialization_mutex_

bool	initialized_ = false

std::thread	audio_publish_thread_

Member Typedef Documentation

◆ Audio

using fp_perception::PerceptionServer::Audio = fp_perception_msgs::msg::PerceptionAudio

◆ ImageAnalysis

using fp_perception::PerceptionServer::ImageAnalysis = fp_perception_msgs::srv::PerceptionImageAnalysis

◆ Sentiment

using fp_perception::PerceptionServer::Sentiment = fp_perception_msgs::srv::PerceptionSentiment

◆ Speech

using fp_perception::PerceptionServer::Speech = fp_perception_msgs::srv::PerceptionSpeech

◆ Transcribe

using fp_perception::PerceptionServer::Transcribe = fp_perception_msgs::srv::PerceptionTranscribe

Constructor & Destructor Documentation

◆ PerceptionServer()

fp_perception::PerceptionServer::PerceptionServer ( const rclcpp::NodeOptions & options = rclcpp::NodeOptions() )

inline

Construct a new Perception Server object.

Parameters

options Node options for the server

◆ ~PerceptionServer()

fp_perception::PerceptionServer::~PerceptionServer ( )

inline

Member Function Documentation

◆ audioCallback()

void fp_perception::PerceptionServer::audioCallback ( const Audio & msg )

inlineprotected

Set the latest audio data to the driver from topic subscription. This function is called when new audio data is received from the subscribed topic.

Parameters

msg	The audio data message received from the topic.

Exceptions

fp_perception_exception if the stream is not active

◆ configure_driver_selection()

void fp_perception::PerceptionServer::configure_driver_selection ( )

inlineprotected

◆ configure_interfaces()

void fp_perception::PerceptionServer::configure_interfaces ( )

inlineprotected

◆ image_analysis_callback()

void fp_perception::PerceptionServer::image_analysis_callback	(	const std::shared_ptr< ImageAnalysis::Request >	request,
		std::shared_ptr< ImageAnalysis::Response >	response
	)

inlineprotected

◆ initialize()

void fp_perception::PerceptionServer::initialize ( )

inline

◆ initialize_pipelines()

void fp_perception::PerceptionServer::initialize_pipelines ( )

inlineprotected

◆ load_driver_if_enabled()

template<typename DriverType >

std::shared_ptr< DriverType > fp_perception::PerceptionServer::load_driver_if_enabled	(	const bool	enabled,
		const DriverSpec &	spec,
		const DriverManager &	driver_manager
	)

inlineprotected

◆ load_drivers()

void fp_perception::PerceptionServer::load_drivers ( const DriverManager & driver_manager )

inlineprotected

◆ make_image_analysis_request()

image_analysis_request fp_perception::PerceptionServer::make_image_analysis_request ( const ImageAnalysis::Request & request )

inlineprotected

◆ make_sentiment_request()

sentiment_request fp_perception::PerceptionServer::make_sentiment_request ( const Sentiment::Request & request )

inlineprotected

◆ make_transcription_request()

transcription_request fp_perception::PerceptionServer::make_transcription_request	(	const Transcribe::Request &	request,
		std_msgs::msg::Header &	header
	)

inlineprotected

◆ publishAudio()

virtual void fp_perception::PerceptionServer::publishAudio ( )

inlineprotectedvirtual

◆ publishVideo()

void fp_perception::PerceptionServer::publishVideo ( )

inlineprotected

◆ read_latest_public_audio()

fp_perception::audio_data fp_perception::PerceptionServer::read_latest_public_audio ( int duration_seconds )

inlineprotected

◆ read_public_audio()

fp_perception::audio_data fp_perception::PerceptionServer::read_public_audio ( const audio_buffer_request & request )

inlineprotected

◆ run_tests()

void fp_perception::PerceptionServer::run_tests ( )

inlineprotected

◆ schedule_deferred_initialization()

void fp_perception::PerceptionServer::schedule_deferred_initialization ( )

inlineprotected

◆ sentiment_callback()

void fp_perception::PerceptionServer::sentiment_callback	(	const std::shared_ptr< Sentiment::Request >	request,
		std::shared_ptr< Sentiment::Response >	response
	)

inlineprotected

Callback function for the sentiment analysis service.

This function is called when a request is received by the sentiment analysis service. It processes the request and sends back a response with the sentiment analysis results.

Parameters

request	The request received from the client
response	The response to be sent back to the client

◆ speech_callback()

void fp_perception::PerceptionServer::speech_callback	(	const std::shared_ptr< Speech::Request >	request,
		std::shared_ptr< Speech::Response >	response
	)

inlineprotected

Service callback for the speech service.

This function is called when a request is made to the speech service.

Parameters

request	The request message containing the audio data and options.
response	The response message to be filled with the transcription result.

◆ start_background_tasks()

void fp_perception::PerceptionServer::start_background_tasks ( )

inlineprotected

◆ stop_background_tasks()

void fp_perception::PerceptionServer::stop_background_tasks ( )

inlineprotected

◆ transcribe_callback()

void fp_perception::PerceptionServer::transcribe_callback	(	const std::shared_ptr< Transcribe::Request >	request,
		std::shared_ptr< Transcribe::Response >	response
	)

inlineprotected

Service callback for transcription requests.

This method handles incoming transcription requests and processes them.

Parameters

request	The incoming transcription request
response	The response to be sent back

◆ write_image_analysis_response()

void fp_perception::PerceptionServer::write_image_analysis_response	(	const image_analysis_result &	result,
		ImageAnalysis::Response &	response
	)

inlineprotected

◆ write_sentiment_response()

void fp_perception::PerceptionServer::write_sentiment_response	(	const sentiment_result &	result,
		Sentiment::Response &	response
	)

inlineprotected

◆ write_transcription_response()

void fp_perception::PerceptionServer::write_transcription_response	(	const transcription_result &	result,
		Transcribe::Response &	response
	)

inlineprotected

Member Data Documentation

◆ audio_input_frame_id_

std::string fp_perception::PerceptionServer::audio_input_frame_id_

protected

◆ audio_input_frequency_

int fp_perception::PerceptionServer::audio_input_frequency_

protected

◆ audio_input_publish_

bool fp_perception::PerceptionServer::audio_input_publish_

protected

◆ audio_input_retention_window_

int fp_perception::PerceptionServer::audio_input_retention_window_

protected

◆ audio_input_topic_

std::string fp_perception::PerceptionServer::audio_input_topic_

protected

◆ audio_output_subscribe_

bool fp_perception::PerceptionServer::audio_output_subscribe_

protected

◆ audio_output_topic_

std::string fp_perception::PerceptionServer::audio_output_topic_

protected

◆ audio_publish_thread_

std::thread fp_perception::PerceptionServer::audio_publish_thread_

protected

◆ audio_publisher_

rclcpp::Publisher<Audio>::SharedPtr fp_perception::PerceptionServer::audio_publisher_

protected

◆ audio_subscriber_

rclcpp::Subscription<Audio>::SharedPtr fp_perception::PerceptionServer::audio_subscriber_

protected

◆ default_audio_request_window_

int fp_perception::PerceptionServer::default_audio_request_window_

protected

◆ deferred_initialize_timer_

rclcpp::TimerBase::SharedPtr fp_perception::PerceptionServer::deferred_initialize_timer_

protected

◆ driver_loader_

pluginlib::ClassLoader<fp_perception::DriverBase> fp_perception::PerceptionServer::driver_loader_

protected

plugin loader for drivers

◆ image_analysis_driver_

std::shared_ptr<fp_perception::ImageAnalysisDriver> fp_perception::PerceptionServer::image_analysis_driver_

protected

shared pointer for image analysis driver

◆ image_analysis_enabled_

bool fp_perception::PerceptionServer::image_analysis_enabled_

protected

◆ image_analysis_pipeline_

std::unique_ptr<ImageAnalysisPipeline> fp_perception::PerceptionServer::image_analysis_pipeline_

protected

◆ image_analysis_service_

rclcpp::Service<ImageAnalysis>::SharedPtr fp_perception::PerceptionServer::image_analysis_service_

protected

◆ image_analysis_service_name_

std::string fp_perception::PerceptionServer::image_analysis_service_name_

protected

◆ image_publisher_

rclcpp::Publisher<sensor_msgs::msg::Image>::SharedPtr fp_perception::PerceptionServer::image_publisher_

protected

◆ initialization_mutex_

std::mutex fp_perception::PerceptionServer::initialization_mutex_

protected

◆ initialized_

bool fp_perception::PerceptionServer::initialized_ = false

protected

◆ kImageAnalysisDriverSpec

constexpr DriverSpec fp_perception::PerceptionServer::kImageAnalysisDriverSpec

inlinestaticconstexpr

Initial value:

{ "image_analysis_driver",
                                                               "fp_perception::OpenAIImageAnalysisDriver",
                                                               "image analysis driver", "ImageAnalysisDriver" }

◆ kMicrophoneDriverSpec

constexpr DriverSpec fp_perception::PerceptionServer::kMicrophoneDriverSpec

inlinestaticconstexpr

Initial value:

{ "microphone_driver", "fp_perception::MicrophoneAudioDriver",

"microphone driver", "AudioSourceDriver" }

◆ kNonRosVisionDriverSpec

constexpr DriverSpec fp_perception::PerceptionServer::kNonRosVisionDriverSpec

inlinestaticconstexpr

Initial value:

{ "non_ros_vision_driver", "fp_perception::OpenCVDriver",

"vision driver", "VisionSourceDriver" }

◆ kRosVisionDriverSpec

constexpr DriverSpec fp_perception::PerceptionServer::kRosVisionDriverSpec

inlinestaticconstexpr

Initial value:

{ "ros_vision_driver", "fp_perception::DefaultDriver",

"vision driver", "VisionSourceDriver" }

◆ kSentimentDriverSpec

constexpr DriverSpec fp_perception::PerceptionServer::kSentimentDriverSpec

inlinestaticconstexpr

Initial value:

{ "sentiment_driver", "fp_perception::SentimentDriver",

"sentiment driver", "SentimentAnalysisDriver" }

◆ kSpeakerDriverSpec

constexpr DriverSpec fp_perception::PerceptionServer::kSpeakerDriverSpec

inlinestaticconstexpr

Initial value:

{ "speaker_driver", "fp_perception::SpeakerAudioDriver",

"speaker driver", "AudioSinkDriver" }

◆ kSpeechDriverSpec

constexpr DriverSpec fp_perception::PerceptionServer::kSpeechDriverSpec

inlinestaticconstexpr

Initial value:

{ "speech_synthesis_driver", "fp_perception::OpenAISpeechDriver",

"speech synthesis driver", "SpeechSynthesisDriver" }

◆ kTranscriptionDriverSpec

constexpr DriverSpec fp_perception::PerceptionServer::kTranscriptionDriverSpec

inlinestaticconstexpr

Initial value:

{ "transcription_driver", "fp_perception::OpenAIDriver",

"transcription driver", "TranscriptionDriver" }

◆ microphone_driver_

std::shared_ptr<fp_perception::AudioSourceDriver> fp_perception::PerceptionServer::microphone_driver_

protected

shared pointer for audio listener driver

◆ non_ros_vision_driver_

std::shared_ptr<fp_perception::VisionSourceDriver> fp_perception::PerceptionServer::non_ros_vision_driver_

protected

shared pointer for vision driver

◆ public_audio_buffer_

AudioBuffer fp_perception::PerceptionServer::public_audio_buffer_

protected

◆ ros_vision_driver_

std::shared_ptr<fp_perception::VisionSourceDriver> fp_perception::PerceptionServer::ros_vision_driver_

protected

shared pointer for vision driver

◆ run_tests_

bool fp_perception::PerceptionServer::run_tests_

protected

Run Tests

◆ sentiment_driver_

std::shared_ptr<fp_perception::SentimentAnalysisDriver> fp_perception::PerceptionServer::sentiment_driver_

protected

shared pointer for sentiment driver

◆ sentiment_driver_mutex_

std::mutex fp_perception::PerceptionServer::sentiment_driver_mutex_

protected

◆ sentiment_enabled_

bool fp_perception::PerceptionServer::sentiment_enabled_

protected

◆ sentiment_pipeline_

std::unique_ptr<SentimentPipeline> fp_perception::PerceptionServer::sentiment_pipeline_

protected

◆ sentiment_service_

rclcpp::Service<Sentiment>::SharedPtr fp_perception::PerceptionServer::sentiment_service_

protected

◆ sentiment_service_name_

std::string fp_perception::PerceptionServer::sentiment_service_name_

protected

◆ speaker_driver_

std::shared_ptr<fp_perception::AudioSinkDriver> fp_perception::PerceptionServer::speaker_driver_

protected

shared pointer for audio speaker driver

◆ speech_driver_

std::shared_ptr<fp_perception::SpeechSynthesisDriver> fp_perception::PerceptionServer::speech_driver_

protected

shared pointer for speech synthesis driver

◆ speech_enabled_

bool fp_perception::PerceptionServer::speech_enabled_

protected

◆ speech_pipeline_

std::unique_ptr<SpeechPipeline> fp_perception::PerceptionServer::speech_pipeline_

protected

◆ speech_service_

rclcpp::Service<Speech>::SharedPtr fp_perception::PerceptionServer::speech_service_

protected

◆ speech_service_name_

std::string fp_perception::PerceptionServer::speech_service_name_

protected

◆ transcription_

rclcpp::Service<Transcribe>::SharedPtr fp_perception::PerceptionServer::transcription_

protected

◆ transcription_driver_

std::shared_ptr<fp_perception::TranscriptionDriver> fp_perception::PerceptionServer::transcription_driver_

protected

shared pointer for transcription driver

◆ transcription_driver_mutex_

std::mutex fp_perception::PerceptionServer::transcription_driver_mutex_

protected

◆ transcription_enabled_

bool fp_perception::PerceptionServer::transcription_enabled_

protected

◆ transcription_pipeline_

std::unique_ptr<TranscriptionPipeline> fp_perception::PerceptionServer::transcription_pipeline_

protected

◆ transcription_service_

std::string fp_perception::PerceptionServer::transcription_service_

protected

◆ use_image_analysis_driver_

bool fp_perception::PerceptionServer::use_image_analysis_driver_

protected

◆ use_microphone_driver_

bool fp_perception::PerceptionServer::use_microphone_driver_

protected

◆ use_non_ros_vision_driver_

bool fp_perception::PerceptionServer::use_non_ros_vision_driver_

protected

◆ use_ros_vision_driver_

bool fp_perception::PerceptionServer::use_ros_vision_driver_

protected

◆ use_sentiment_driver_

bool fp_perception::PerceptionServer::use_sentiment_driver_

protected

◆ use_speaker_driver_

bool fp_perception::PerceptionServer::use_speaker_driver_

protected

◆ use_speech_driver_

bool fp_perception::PerceptionServer::use_speech_driver_

protected

◆ use_transcription_driver_

bool fp_perception::PerceptionServer::use_transcription_driver_

protected

◆ vision_input_frame_id_

std::string fp_perception::PerceptionServer::vision_input_frame_id_

protected

◆ vision_input_frequency_

int fp_perception::PerceptionServer::vision_input_frequency_

protected

◆ vision_input_publish_

bool fp_perception::PerceptionServer::vision_input_publish_

protected

◆ vision_input_topic_

std::string fp_perception::PerceptionServer::vision_input_topic_

protected

The documentation for this class was generated from the following file:

fp_perception/include/fp_perception/perception_server.hpp

Static Public Attributes
static constexpr DriverSpec	kRosVisionDriverSpec

static constexpr DriverSpec	kNonRosVisionDriverSpec

static constexpr DriverSpec	kMicrophoneDriverSpec

static constexpr DriverSpec	kSpeakerDriverSpec

static constexpr DriverSpec	kTranscriptionDriverSpec

static constexpr DriverSpec	kSpeechDriverSpec

static constexpr DriverSpec	kSentimentDriverSpec

static constexpr DriverSpec	kImageAnalysisDriverSpec

Classes

Public Types

Public Member Functions

Static Public Attributes

Protected Member Functions

Protected Attributes

Member Typedef Documentation

◆ Audio

◆ ImageAnalysis

◆ Sentiment

◆ Speech

◆ Transcribe

Constructor & Destructor Documentation

◆ PerceptionServer()

◆ ~PerceptionServer()

Member Function Documentation

◆ audioCallback()

◆ configure_driver_selection()

◆ configure_interfaces()

◆ image_analysis_callback()

◆ initialize()

◆ initialize_pipelines()

◆ load_driver_if_enabled()

◆ load_drivers()

◆ make_image_analysis_request()

◆ make_sentiment_request()

◆ make_transcription_request()

◆ publishAudio()

◆ publishVideo()

◆ read_latest_public_audio()

◆ read_public_audio()

◆ run_tests()

◆ schedule_deferred_initialization()

◆ sentiment_callback()

◆ speech_callback()

◆ start_background_tasks()

◆ stop_background_tasks()

◆ transcribe_callback()

◆ write_image_analysis_response()

◆ write_sentiment_response()

◆ write_transcription_response()

Member Data Documentation

◆ audio_input_frame_id_

◆ audio_input_frequency_

◆ audio_input_publish_

◆ audio_input_retention_window_

◆ audio_input_topic_

◆ audio_output_subscribe_

◆ audio_output_topic_

◆ audio_publish_thread_

◆ audio_publisher_

◆ audio_subscriber_

◆ default_audio_request_window_

◆ deferred_initialize_timer_

◆ driver_loader_

◆ image_analysis_driver_

◆ image_analysis_enabled_

◆ image_analysis_pipeline_

◆ image_analysis_service_

◆ image_analysis_service_name_

◆ image_publisher_

◆ initialization_mutex_

◆ initialized_

◆ kImageAnalysisDriverSpec

◆ kMicrophoneDriverSpec

◆ kNonRosVisionDriverSpec

◆ kRosVisionDriverSpec

◆ kSentimentDriverSpec

◆ kSpeakerDriverSpec

◆ kSpeechDriverSpec

◆ kTranscriptionDriverSpec

◆ microphone_driver_

◆ non_ros_vision_driver_

◆ public_audio_buffer_

◆ ros_vision_driver_

◆ run_tests_

◆ sentiment_driver_

◆ sentiment_driver_mutex_

◆ sentiment_enabled_

◆ sentiment_pipeline_