package com.cr.pages import android.graphics.ImageFormat import android.graphics.Rect import android.graphics.YuvImage import android.media.MediaCodecInfo import android.media.MediaFormat import android.os.Bundle import android.view.LayoutInflater import android.view.SurfaceHolder import android.view.SurfaceView import android.view.View import android.view.ViewGroup import android.view.ViewTreeObserver.OnGlobalLayoutListener import android.widget.Button import android.widget.TextView import androidx.constraintlayout.widget.ConstraintLayout import androidx.fragment.app.activityViewModels import com.cr.common.CrUnitManager import com.cr.cruav.CrApplication import com.cr.cruav.R import com.cr.data.CrUtil import com.cr.event.CrCommonAction import com.cr.event.EventCommon import com.cr.models.CompletionModel import com.cr.models.ICompletion import com.cr.viewmodel.CrFlightControlVM import com.cr.viewmodel.CrLiveStreamVM import com.cr.viewmodel.CrVideoChannelListener import com.cr.viewmodel.CrVideoChannelVM import com.squareup.otto.Subscribe import dji.v5.common.video.channel.VideoChannelType import dji.v5.common.video.decoder.DecoderOutputMode import dji.v5.common.video.decoder.DecoderState import dji.v5.common.video.decoder.VideoDecoder import dji.v5.common.video.interfaces.* import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.launch import java.io.File import java.io.FileNotFoundException import java.io.FileOutputStream import java.io.IOException import java.io.OutputStream /** * 操作系统:MAC系统 * 创建者:王成 * 创建日期:2023/3/10 08:50 * 描述:图传窗口 */ class FragmentFPV : CrAnimationFragment(), SurfaceHolder.Callback, OnGlobalLayoutListener { /** * FPV窗口监听 */ interface FPVListener { // todo: 2023/3/13 切换窗口 fun triggerWindow() } // define: 2023/3/10 延迟初始化一个通道模型 private val videoChannelVM: CrVideoChannelVM by activityViewModels() // define: 2023/3/10 将飞行器视图绑定到模型 private val flightControlVm: CrFlightControlVM by activityViewModels() // define: 2023/9/12 绑定直播模型 private val liveStreamVm: CrLiveStreamVM by activityViewModels() // define: 2023/3/10 延迟初始化一个视频画布 private lateinit var surfaceView: SurfaceView private var btnTrigger: Button? = null private var lblInfo: TextView? = null // define: 2023/8/4 显示信息 private var panelFpv: ConstraintLayout? = null // define: 2023/9/14 容器 // define: 2023/3/11 视频通道 private var curVideoChannel: IVideoChannel? = null // define: 2023/3/10 定义视频解码器 private var videoDecoder: IVideoDecoder? = null // todo: 2023/8/4 视频相关参数 private var videoWidth: Int = -1 private var videoHeight: Int = -1 private var widthChanged: Boolean = false private var heightChange: Boolean = false private var fps: Int = -1 // todo: 2023/9/14 画布相关参数 private var surfaceViewWidth: Int = 0 private var surfaceViewHeight: Int = 0 // todo: 2023/3/13 监听 lateinit var listener: FPVListener /** * 创建视图 * @param inflater LayoutInflater * @param container ViewGroup? * @param savedInstanceState Bundle? * @return View? */ override fun onCreateView( inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle? ): View? { var view = inflater.inflate(R.layout.frag_fpv, container, false) // todo: 2023/8/4 关闭 View Layer。 View Layer 可以加速无 invalidate() 时的刷新效率,但对于需要调用 invalidate() 的刷新无法加速 view.setLayerType(View.LAYER_TYPE_NONE, null) // todo: 2023/3/13 给View挂接事件 view.setOnClickListener(clickListener) // todo: 2023/3/10 挂接图传视图控件 surfaceView = view.findViewById(R.id.surface_fpv) btnTrigger = view.findViewById(R.id.fpv_trigger) btnTrigger?.setOnClickListener(clickListener) // todo: 2023/9/14 挂接容器 panelFpv = view.findViewById(R.id.fpv_panel) // todo: 2023/8/4 挂接信息显示组件 lblInfo = view.findViewById(R.id.fpv_info) // todo: 2023/3/10 设置视图控件监听 surfaceView.holder.addCallback(this) // todo: 2023/8/7 注册事件监听 CrApplication.getEventBus().register(this) return view } /** * 添加视图尺寸变化监听 */ private fun addSizeChangeForSurfaceView() { surfaceView.viewTreeObserver!!.addOnGlobalLayoutListener(this) } /** * 画布尺寸变化监听 */ override fun onGlobalLayout() { if (surfaceViewWidth != surfaceView.width && surfaceViewHeight != surfaceView.height) { surfaceViewWidth = surfaceView.width surfaceViewHeight = surfaceView.height CrUtil.print("宽度:${surfaceViewWidth} 高度:${surfaceViewHeight}") // todo: 2023/9/14 移除监听 要不会多次调用 surfaceView.viewTreeObserver.removeOnGlobalLayoutListener(this) } } /** * 视图创建完成后调用 * @param view View * @param savedInstanceState Bundle? */ override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) // todo: 2023/3/11 初始化 init() } /** * 点击事件 */ private var clickListener = View.OnClickListener { when (it.id) { R.id.fpv_trigger -> { // todo: 2023/3/13 切换视频通道 videoChannelVM.triggerStreamSource() } R.id.fpv_panel -> { // todo: 2023/3/13 切换窗口 listener.triggerWindow() // // todo: 2023/9/14 添加画布尺寸变化监听 // // todo: 2023/9/14 主要是为了在画布变化时可以调整解码器解码的宽度和高度 // addSizeChangeForSurfaceView() // todo: 2023/9/15 判断当前是否在直播 如果是 则先停止 再开启 if (liveStreamVm?.streamStatusInfo.value!!.isStreaming) { liveStreamVm.stopStream(object : ICompletion { // todo: 2023/9/15 停止完成后回调 override fun onCompletion(completion: CompletionModel) { if (completion.isSuccess == true) liveStreamVm?.startStream(null) } }) } } } } /** * 初始化 */ private fun init() { // todo: 2023/8/4 注册订阅 videoChannelVM.videoChannelInfo.observe(viewLifecycleOwner) { it?.let { val videoStreamInfo = "帧率:[${it.fps}] 宽:[${videoWidth}] 高:[${videoHeight}]" lblInfo?.text = videoStreamInfo } } // todo: 2023/3/11 绑定视频通道变化监听 videoChannelVM.listener = object : CrVideoChannelListener { // todo: 2023/3/13 变更通道 override fun initVideoChannel(videoChannel: IVideoChannel) { curVideoChannel = videoChannel // todo: 2023/8/4 添加组针变化监听 curVideoChannel?.addStreamDataListener(streamDataListener) setChannelToSurface() } // todo: 2023/3/13 更新通道类型 override fun updateVideoChannelType(channelType: VideoChannelType) { changeVideoDecoder(channelType) } } // todo: 2023/3/11 订阅飞行器 flightControlVm.flightControlInfo.observe(requireActivity()) { it?.let { if (it.isConnection) { videoChannelVM.beginVideoChannel() } else { videoChannelVM.endVideoChannel() } } } } /** * 变更通道类型 * @param channelType VideoChannelType */ private fun changeVideoDecoder(channelType: VideoChannelType) { videoDecoder?.videoChannelType = channelType surfaceView.invalidate() } /** * 设置视频通道到画布 */ private fun setChannelToSurface() { // todo: 2023/9/15 释放解码器资源 videoDecoder?.let { videoDecoder?.removeDecoderStateChangeListener(decoderStateChangeListener) videoDecoder?.onPause() videoDecoder?.destroy() videoDecoder = null } // todo: 2023/9/15 重新判断初始化 if (videoDecoder == null) { curVideoChannel?.let { videoDecoder = VideoDecoder( this@FragmentFPV.context, it.videoChannelType, DecoderOutputMode.SURFACE_MODE, surfaceView.holder, surfaceView.width, surfaceView.height, true ) // todo: 2023/9/12 设置直播视频通道 // liveStreamVm.setVideoChannel(it.videoChannelType) // todo: 2023/3/10 添加解码状态监听 videoDecoder?.addDecoderStateChangeListener(decoderStateChangeListener) } } else if (videoDecoder?.decoderStatus == DecoderState.PAUSED) { videoDecoder?.onResume() } } /** * 解码器状态监听 */ private val decoderStateChangeListener = DecoderStateChangeListener { oldState, newState -> mainHandler.post { // todo: 2023/3/10 更新编码状态 videoChannelVM.videoChannelInfo.value?.decoderState = newState // todo: 2023/3/10 刷新数据 videoChannelVM.refreshVideoChannelInfo() } } /** * 组针后数据的监听 */ private val streamDataListener = StreamDataListener { it?.let { if (fps != it.fps) { fps = it.fps mainHandler.post { videoChannelVM.videoChannelInfo.value?.fps = fps videoChannelVM.refreshVideoChannelInfo() } } if (videoWidth != it.width) { videoWidth = it.width widthChanged = true } if (videoHeight != it.height) { videoHeight = it.height heightChange = true } if (widthChanged || heightChange) { widthChanged = false heightChange = false mainHandler.post { videoChannelVM.videoChannelInfo.value?.resolution = "${videoWidth}*${videoHeight}" videoChannelVM.refreshVideoChannelInfo() } } } } /** * surfaceView 创建监听 * @param p0 SurfaceHolder */ override fun surfaceCreated(p0: SurfaceHolder) { setChannelToSurface() } /** * surfaceView变化监听 * @param p0 SurfaceHolder * @param p1 Int * @param p2 Int * @param p3 Int */ override fun surfaceChanged(p0: SurfaceHolder, p1: Int, p2: Int, p3: Int) { setChannelToSurface() } /** * surfaceView释放监听 * @param p0 SurfaceHolder */ override fun surfaceDestroyed(p0: SurfaceHolder) { // todo: 2023/3/10 停止解码 videoDecoder?.onPause() } /** * 释放资源 */ private fun releaseResource() { // todo: 2023/8/7 解除监听 curVideoChannel?.removeStreamDataListener(streamDataListener) videoDecoder?.removeDecoderStateChangeListener(decoderStateChangeListener) videoDecoder?.removeYuvDataListener(yuvDataListener) // todo: 2023/3/10 释放编码资源 if (videoDecoder != null) { videoDecoder?.destroy() videoDecoder = null } } /** * 覆写视图释放 */ override fun onDestroyView() { super.onDestroyView() // todo: 2023/8/7 释放资源 releaseResource() // todo: 2023/8/7 解除事件监听 CrApplication.getEventBus().unregister(this) } /** * 生命周期函数 * 转入前台可见 */ override fun onResume() { super.onResume() curVideoChannel?.let { // todo: 2023/8/7 添加监听 it.addStreamDataListener(streamDataListener) // todo: 2023/8/14 启用图传 handlerYUV(true) } } /** * 转入后台不可见 */ override fun onStop() { super.onStop() // todo: 2023/8/7 释放资源 releaseResource() } /** * 模式切换 true表示切换到图传模式 false表示切换到YUV模式 * @param isSelected Boolean */ private fun handlerYUV(isSelected: Boolean) { if (!isSelected) { // todo: 2023/8/7 如果解码器存在 则停用并置null videoDecoder?.let { videoDecoder!!.onPause() videoDecoder!!.destroy() videoDecoder = null } // todo: 2023/8/7 重新定义解码器 videoDecoder = VideoDecoder(this@FragmentFPV.context, curVideoChannel!!.videoChannelType) videoDecoder?.addDecoderStateChangeListener(decoderStateChangeListener) videoDecoder?.addYuvDataListener(yuvDataListener) } else { // todo: 2023/8/7 如果解码器存在 则停用并置null videoDecoder?.let { videoDecoder!!.onPause() videoDecoder!!.destroy() videoDecoder = null } // todo: 2023/8/7 重新定义解码器 videoDecoder = VideoDecoder( this@FragmentFPV.context, curVideoChannel!!.videoChannelType, DecoderOutputMode.SURFACE_MODE, surfaceView.holder ) videoDecoder?.addDecoderStateChangeListener(decoderStateChangeListener) videoDecoder?.removeYuvDataListener(yuvDataListener) } } /** * yuv数据监听 */ private val yuvDataListener = YuvDataListener { mediaFormat, data, width, height -> // todo: 2023/8/7 创建立即执行的协程 data?.let { GlobalScope.launch { // todo: 2023/8/7 执行一次保存 就恢复图传 handlerYUV(true) // todo: 2023/8/7 保存图片 saveYuvData(mediaFormat, data, width, height) } } } /** * 保存当前帧Yuv数据 * @param mediaFormat MediaFormat? 媒体格式 * @param data ByteArray? 数据 * @param width Int 宽度 * @param height Int 高度 */ private fun saveYuvData(mediaFormat: MediaFormat?, data: ByteArray?, width: Int, height: Int) { data?.let { mediaFormat?.let { when (it.getInteger(MediaFormat.KEY_COLOR_FORMAT)) { MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar -> { newSaveYuvDataToJPEG420P(data, width, height) } MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar -> { newSaveYuvDataToJPEG(data, width, height) } } } ?: sendFailureMessage("数据格式异常") } ?: sendFailureMessage("无图像数据") } /** * 发送错误消息事件 * @param error String 错误消息 */ private fun sendFailureMessage(error: String) { CrApplication.getEventBus() .post(EventCommon(CrCommonAction.VIDEO_SAVE_YUV_FAILURE, error)) } /** * 发送YUA数据保存成功消息 * @param fileName String 保存的文件名称 */ private fun sendSaveYUVSuccessMessage(fileName: String) { CrApplication.getEventBus() .post(EventCommon(CrCommonAction.VIDEO_SAVE_YUV_SUCCESS, fileName)) } /** * 保存yuv数据到JPEG420P * @param yuvFrame ByteArray 帧数据 * @param width Int 宽度 * @param height Int 高度 */ private fun newSaveYuvDataToJPEG420P(yuvFrame: ByteArray, width: Int, height: Int) { if (yuvFrame.size < width * height) { sendFailureMessage("帧数据异常") return } val length = width * height val u = ByteArray(width * height / 4) val v = ByteArray(width * height / 4) for (i in u.indices) { u[i] = yuvFrame[length + i] v[i] = yuvFrame[length + u.size + i] } for (i in u.indices) { yuvFrame[length + 2 * i] = v[i] yuvFrame[length + 2 * i + 1] = u[i] } screenShot( yuvFrame, width, height ) } /** * 保存Yuv数据到JPEG * @param yuvFrame ByteArray 帧数据 * @param width Int 宽度 * @param height Int 高度 */ private fun newSaveYuvDataToJPEG(yuvFrame: ByteArray, width: Int, height: Int) { if (yuvFrame.size < width * height) { sendFailureMessage("帧数据异常") return } val length = width * height val u = ByteArray(width * height / 4) val v = ByteArray(width * height / 4) for (i in u.indices) { v[i] = yuvFrame[length + 2 * i] u[i] = yuvFrame[length + 2 * i + 1] } for (i in u.indices) { yuvFrame[length + 2 * i] = u[i] yuvFrame[length + 2 * i + 1] = v[i] } screenShot( yuvFrame, width, height ) } /** * 保存图片 * @param buf ByteArray 数据 * @param width Int 宽度 * @param height Int 高度 */ private fun screenShot(buf: ByteArray, width: Int, height: Int) { // todo: 2023/8/7 获取图片 val yuvImage = YuvImage(buf, ImageFormat.NV21, width, height, null) // todo: 2023/8/7 保存文件 val outputFile: OutputStream var fileName = "uav_${CrUnitManager.toSystemDate()}.jpg" var path = "${CrUtil.IMAGE_PATH}${fileName}" outputFile = try { FileOutputStream(File(path)) } catch (e: FileNotFoundException) { sendFailureMessage("异常:${e.message}") return } // todo: 2023/8/7 压缩保存图片 yuvImage.compressToJpeg(Rect(0, 0, width, height), 100, outputFile) try { outputFile.close() // todo: 2023/8/7 发送正确消息 sendSaveYUVSuccessMessage(fileName) } catch (e: IOException) { sendFailureMessage("IO错误:${e.message}") } } /** * 设置画布尺寸变化 */ fun setSurfaceViewSizeChange() { var width = surfaceView.width var height = surfaceView.height CrUtil.print("宽度:${width} 高度:${height}") } // todo: 2023/8/7 订阅事件 @Subscribe fun onChanged(event: EventCommon) { when (event.action) { CrCommonAction.VIDEO_SAVE_YUV -> { // todo: 2023/8/7 切换到YUV模式 handlerYUV(false) } } } }