技术背景
VR（虚拟现实技术）给我们带来身临其境的视觉体验，广泛的应用于城市规划、教育培训、工业仿真、房地产、水利电力、室内设计、文旅、军事等众多领域，常用的行业比如：
教育行业：VR头显可以用于教育培训，提供沉浸式的教学体验，例如虚拟实验室、虚拟课堂等，帮助学生更好地理解和掌握知识。医疗行业：VR头显可以用于医疗训练和治疗，例如手术模拟、康复训练等，提高医疗效果和质量。文旅行业：VR头显可以用于旅游娱乐，提供沉浸式的旅游体验，例如虚拟旅游、文化遗产展示等。房地产行业：VR头显可以用于房地产展示，提供更加真实、直观的房屋展示和体验，帮助客户更好地了解和选择房屋。展览展示行业：VR头显可以用于展览展示，提供沉浸式的展览体验，例如虚拟展厅、虚拟展品等，吸引观众的注意和参与。军事行业：VR头显可以用于军事训练和作战指挥，提供更加真实、逼真的军事训练环境。
技术实现
如何在VR头显实现RTMP或RTSP播放？
VR头显播放RTMP或RTSP流数据，简单来说，通过jni层打通RTMP或RTSP流传输，解包并解码回调给Unity YUV或RGB数据，Unity场景下，绘制即可，本文以大牛直播SDK的Unity平台RTMP、RTSP播放为例，介绍下具体技术实现：
开始播放：
public void Play()
{
if (is_running)
{
Debug.Log("已经在播放。。");
return;
}
//获取输入框的url
string url = input_url_.text.Trim();
OpenPlayer();
if ( player_handle_ == 0 )
return;
NT_U3D_Set_Game_Object(player_handle_, game_object_);
/* ++ 播放前参数配置可加在此处 ++ */
int is_using_tcp = 0; //TCP/UDP模式设置
NT_U3D_SetRTSPTcpMode(player_handle_, is_using_tcp);
int is_report = 0;
int report_interval = 1;
NT_U3D_SetReportDownloadSpeed(player_handle_, is_report, report_interval); //下载速度回调
NT_U3D_SetBuffer(player_handle_, play_buffer_time_); //设置buffer time
NT_U3D_SetPlayerLowLatencyMode(player_handle_, is_low_latency_ ? 1 : 0); //设置是否启用低延迟模式
NT_U3D_SetMute(player_handle_, is_mute_ ? 1 : 0); //是否启动播放的时候静音
NT_U3D_SetAudioVolume(player_handle_, cur_audio_volume_); //设置播放音量
NT_U3D_SetVideoDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0); //设置H.264软硬解模式
NT_U3D_SetVideoHevcDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0); //设置H.265软硬解模式
int is_fast_startup = 1;
NT_U3D_SetFastStartup(player_handle_, is_fast_startup); //设置快速启动模式
int rtsp_timeout = 10;
NT_U3D_SetRTSPTimeout(player_handle_, rtsp_timeout); //设置RTSP超时时间
int is_auto_switch_tcp_udp = 1;
NT_U3D_SetRTSPAutoSwitchTcpUdp(player_handle_, is_auto_switch_tcp_udp); //设置TCP/UDP模式自动切换
int is_audiotrack = 1;
NT_U3D_SetAudioOutputType(player_handle_, is_audiotrack); //设置音频输出模式: if 0: 自动选择; if with 1: audiotrack模式
NT_U3D_SetUrl(player_handle_, videoUrl);
/* -- 播放前参数配置可加在此处 -- */
int flag = NT_U3D_StartPlay(player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
is_need_get_frame_ = true;
Debug.Log("播放成功");
}
else
{
is_need_get_frame_ = false;
Debug.LogError("播放失败");
}
is_running = true;
}
Close Player：
private void ClosePlayer()
{
is_need_get_frame_ = false;
is_need_init_texture_ = false;
int flag = NT_U3D_StopPlay(player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
Debug.Log("停止成功");
}
else
{
Debug.LogError("停止失败");
}
flag = NT_U3D_Close(player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
Debug.Log("关闭成功");
}
else
{
Debug.LogError("关闭失败");
}
player_handle_ = 0;
NT_U3D_UnInit();
is_running = false;
video_width_ = 0;
video_height_ = 0;
}
Event事件回调处理：
///
/// android 传递过来 code
///
///
public void onNTSmartEvent(string event_message)
{
if (null == event_message || event_message.Length < 1)
return;
string[] strs = event_message.Split(',');
if (null == strs || strs.Length < 6)
return;
string player_handle =strs[0];
string code = strs[1];
string param1 = strs[2];
string param2 = strs[3];
string param3 = strs[4];
string param4 = strs[5];
Debug.Log("[daiusdk] code: 0x" + Convert.ToString(Convert.ToInt32(code), 16));
String player_event = "";
switch (Convert.ToInt32(code))
{
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STARTED:
player_event = "开始..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING:
player_event = "连接中..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED:
player_event = "连接失败..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED:
player_event = "连接成功..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED:
player_event = "连接断开..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP:
player_event = "停止播放..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO:
player_event = "分辨率信息: width: " + Convert.ToInt32(param1) + ", height: " + Convert.ToInt32(param2);
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED:
player_event = "收不到媒体数据，可能是url错误..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL:
player_event = "切换播放URL..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE:
player_event = "快照: " + param1 + " 路径：" + param3;
if (Convert.ToInt32(param1) == 0)
{
player_event = "截取快照成功..";
}
else
{
player_event = "截取快照失败..";
}
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE:
player_event = "[record]开始一个新的录像文件 : " + param3;
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED:
player_event = "[record]已生成一个录像文件 : " + param3;
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING:
player_event = "Start_Buffering..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING:
player_event = "Buffering: " + Convert.ToInt32(param1);
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING:
player_event = "Stop_Buffering..";
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED:
player_event = "download_speed:" + param1 + "Byte/s" + ", "
+ (Convert.ToInt32(param1) * 8 / 1000) + "kbps" + ", " + (Convert.ToInt32(param1) / 1024)
+ "KB/s";
break;
}
Debug.Log(player_event);
player_event = null;
strs = null;
}
如何封装实现原生jni层交互：
/// SmartPlayerAndroidMono.cs
/// Author: daniusdk.com
///Created on 2018/05/10
///
/// Init
///
public int NT_U3D_Init()
{
return player_obj_.Call
}
///
/// 开始
/// 返回播放句柄
///
public long NT_U3D_Open()
{
return player_obj_.Call
}
///
/// Register Game Object,用于消息传递
///
public int NT_U3D_Set_Game_Object(long handle, string gameObjectName)
{
return player_obj_.Call
}
///
/// 设置H.264解码方式 false 软件解码 true 硬件解码 默认为false
///
///
public int NT_U3D_SetVideoDecoderMode(long handle, int isHwDecoder)
{
return player_obj_.Call
}
///
/// 设置H.265 解码方式 false 软件解码 true 硬件解码 默认为false
///
///
public int NT_U3D_SetVideoHevcDecoderMode(long handle, int isHevcHwDecoder)
{
return player_obj_.Call
}
///
/// 设置音频输出模式: if 0: 自动选择; if with 1: audiotrack模式
///
///
public int NT_U3D_SetAudioOutputType(long handle, int use_audiotrack)
{
return player_obj_.Call
}
///
/// 设置播放端缓存大小, 默认200毫秒
///
///
public int NT_U3D_SetBuffer(long handle, int buffer)
{
return player_obj_.Call
}
///
/// 接口可实时调用：设置是否实时静音，1：静音; 0: 取消静音
///
///
public int NT_U3D_SetMute(long handle, int is_mute)
{
return player_obj_.Call
}
///
/// 接口可实时调用：设置播放音量，范围是[0, 100], 0是静音，100是最大音量, 默认是100
///
///
public int NT_U3D_SetAudioVolume(long handle, int audio_volume)
{
return player_obj_.Call
}
///
/// 设置RTSP TCP模式, 1: TCP; 0: UDP
///
///
public int NT_U3D_SetRTSPTcpMode(long handle, int is_using_tcp)
{
return player_obj_.Call
}
///
/// 设置RTSP超时时间, timeout单位为秒，必须大于0
///
///
public int NT_U3D_SetRTSPTimeout(long handle, int timeout)
{
return player_obj_.Call
}
///
/// 设置RTSP TCP/UDP自动切换
/// NOTE: 对于RTSP来说，有些可能支持rtp over udp方式，有些可能支持使用rtp over tcp方式.
/// 为了方便使用，有些场景下可以开启自动尝试切换开关, 打开后如果udp无法播放，sdk会自动尝试tcp, 如果tcp方式播放不了,sdk会自动尝试udp.
///
///
/// timeout：如果设置1的话, sdk将在tcp和udp之间尝试切换播放，如果设置为0，则不尝试切换.
public int NT_U3D_SetRTSPAutoSwitchTcpUdp(long handle, int is_auto_switch_tcp_udp)
{
return player_obj_.Call
}
///
/// 设置快速启动该模式，
///
///
public int NT_U3D_SetFastStartup(long handle, int is_fast_startup)
{
return player_obj_.Call
}
///
/// 设置超低延迟模式 false不开启 true开启 默认false
///
///
public int NT_U3D_SetPlayerLowLatencyMode(long handle, int mode)
{
return player_obj_.Call
}
///
/// 设置视频垂直反转
/// is_flip： 0: 不反转, 1: 反转
///
///
public int NT_U3D_SetFlipVertical(long handle, int is_flip)
{
return player_obj_.Call
}
///
/// 设置视频水平反转
/// is_flip： 0: 不反转, 1: 反转
///
///
public int NT_U3D_SetFlipHorizontal(long handle, int is_flip)
{
return player_obj_.Call
}
///
/// 设置顺时针旋转, 注意除了0度之外， 其他角度都会额外消耗性能
/// degress： 当前支持 0度，90度, 180度, 270度 旋转
///
///
public int NT_U3D_SetRotation(long handle, int degress)
{
return player_obj_.Call
}
///
/// 设置是否回调下载速度
/// is_report: if 1: 上报下载速度, 0: 不上报.
/// report_interval: 上报间隔，以秒为单位，>0.
///
///
///
public int NT_U3D_SetReportDownloadSpeed(long handle, int is_report, int report_interval)
{
return player_obj_.Call
}
///
/// 设置是否需要在播放或录像过程中快照
///
///
public int NT_U3D_SetSaveImageFlag(long handle, int is_save_image)
{
return player_obj_.Call
}
///
/// 播放或录像过程中快照
///
///
public int NT_U3D_SaveCurImage(long handle, string imageName)
{
return player_obj_.Call
}
///
/// 播放或录像过程中，快速切换url
///
///
public int NT_U3D_SwitchPlaybackUrl(long handle, string uri)
{
return player_obj_.Call
}
///
/// 创建录像存储路径
///
///
public int NT_U3D_CreateFileDirectory(string path)
{
return player_obj_.Call
}
///
/// 设置录像存储路径
///
///
public int NT_U3D_SetRecorderDirectory(long handle, string path)
{
return player_obj_.Call
}
///
/// 设置单个录像文件大小
///
///
public int NT_U3D_SetRecorderFileMaxSize(long handle, int size)
{
return player_obj_.Call
}
///
/// 设置录像时音频转AAC编码的开关
/// aac比较通用，sdk增加其他音频编码(比如speex, pcmu, pcma等)转aac的功能.
/// 注意: 转码会增加性能消耗
///
///
/// is_transcode：设置为1的话，如果音频编码不是aac，则转成aac，如果是aac，则不做转换. 设置为0的话，则不做任何转换. 默认是0.
public int NT_U3D_SetRecorderAudioTranscodeAAC(long handle, int is_transcode)
{
return player_obj_.Call
}
///
/// 设置播放路径
///
public int NT_U3D_SetUrl(long handle, string url)
{
return player_obj_.Call
}
///
/// 开始播放
///
public int NT_U3D_StartPlay(long handle)
{
return player_obj_.Call
}
///
/// 获取YUV数据
///
public AndroidJavaObject NT_U3D_GetVideoFrame(long handle)
{
return player_obj_.Call
}
///
/// 停止播放
///
public int NT_U3D_StopPlay(long handle)
{
return player_obj_.Call
}
///
/// 开始录像
///
public int NT_U3D_StartRecorder(long handle)
{
return player_obj_.Call
}
///
/// 停止录像
///
public int NT_U3D_StopRecorder(long handle)
{
return player_obj_.Call
}
///
/// 关闭播放
///
public int NT_U3D_Close(long handle)
{
return player_obj_.Call
}
///
/// UnInit Player
///
public int NT_U3D_UnInit()
{
return DANIULIVE_RETURN_OK;
}
技术总结
通过实际测试来看，VR头显端，如果设备性能尚可的话，播放RTMP或RTSP，可实现毫秒级的延迟，可满足大多数有交互诉求的技术场景，此外，如果头显端支持硬解码的话，可以优先考虑硬解码。
好文推荐
您阅读本篇文章共花了：
发表评论