iOS 8에서 RTP H264 용 AVSampleBufferDisplayLayer를 사용하는 방법 GStreamer로 스트림 하시겠습니까?

Question

iOS 8의 프로그래머가 사용할 수있는 HW-H264 디코더를 알게 된 후 지금 사용하고 싶습니다. WWDC 2014에서 '비디오 인코딩 및 디코딩에 직접 액세스'에 대한 훌륭한 소개가 있습니다. 너는 here을 볼 수있다.

사례 1을 바탕으로, GStreamer에서 H264-RTP-UDP-Stream을 가져올 수있는 응용 프로그램을 개발하기 시작했습니다. NAL에 직접 액세스하려면 'appsink'- 요소로 싱크합니다. 단위와 변환을 수행하여 CMSampleBuffers를 생성합니다. 이는 AVSampleBufferDisplayLayer이 표시 할 수 있습니다.

모든 일을 코드의 흥미로운 부분은 다음과 같다 :

// 
// GStreamerBackend.m 
// 

#import "GStreamerBackend.h" 

NSString * const naluTypesStrings[] = { 
    @"Unspecified (non-VCL)", 
    @"Coded slice of a non-IDR picture (VCL)", 
    @"Coded slice data partition A (VCL)", 
    @"Coded slice data partition B (VCL)", 
    @"Coded slice data partition C (VCL)", 
    @"Coded slice of an IDR picture (VCL)", 
    @"Supplemental enhancement information (SEI) (non-VCL)", 
    @"Sequence parameter set (non-VCL)", 
    @"Picture parameter set (non-VCL)", 
    @"Access unit delimiter (non-VCL)", 
    @"End of sequence (non-VCL)", 
    @"End of stream (non-VCL)", 
    @"Filler data (non-VCL)", 
    @"Sequence parameter set extension (non-VCL)", 
    @"Prefix NAL unit (non-VCL)", 
    @"Subset sequence parameter set (non-VCL)", 
    @"Reserved (non-VCL)", 
    @"Reserved (non-VCL)", 
    @"Reserved (non-VCL)", 
    @"Coded slice of an auxiliary coded picture without partitioning (non-VCL)", 
    @"Coded slice extension (non-VCL)", 
    @"Coded slice extension for depth view components (non-VCL)", 
    @"Reserved (non-VCL)", 
    @"Reserved (non-VCL)", 
    @"Unspecified (non-VCL)", 
    @"Unspecified (non-VCL)", 
    @"Unspecified (non-VCL)", 
    @"Unspecified (non-VCL)", 
    @"Unspecified (non-VCL)", 
    @"Unspecified (non-VCL)", 
    @"Unspecified (non-VCL)", 
    @"Unspecified (non-VCL)", 
}; 


static GstFlowReturn new_sample(GstAppSink *sink, gpointer user_data) 
{ 
    GStreamerBackend *backend = (__bridge GStreamerBackend *)(user_data); 
    GstSample *sample = gst_app_sink_pull_sample(sink); 
    GstBuffer *buffer = gst_sample_get_buffer(sample); 
    GstMemory *memory = gst_buffer_get_all_memory(buffer); 

    GstMapInfo info; 
    gst_memory_map (memory, &info, GST_MAP_READ); 

    int startCodeIndex = 0; 
    for (int i = 0; i < 5; i++) { 
     if (info.data[i] == 0x01) { 
      startCodeIndex = i; 
      break; 
     } 
    } 
    int nalu_type = ((uint8_t)info.data[startCodeIndex + 1] & 0x1F); 
    NSLog(@"NALU with Type \"%@\" received.", naluTypesStrings[nalu_type]); 
    if(backend.searchForSPSAndPPS) { 
     if (nalu_type == 7) 
      backend.spsData = [NSData dataWithBytes:&(info.data[startCodeIndex + 1]) length: info.size - 4]; 

     if (nalu_type == 8) 
      backend.ppsData = [NSData dataWithBytes:&(info.data[startCodeIndex + 1]) length: info.size - 4]; 

     if (backend.spsData != nil && backend.ppsData != nil) { 
      const uint8_t* const parameterSetPointers[2] = { (const uint8_t*)[backend.spsData bytes], (const uint8_t*)[backend.ppsData bytes] }; 
      const size_t parameterSetSizes[2] = { [backend.spsData length], [backend.ppsData length] }; 

      CMVideoFormatDescriptionRef videoFormatDescr; 
      OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault, 2, parameterSetPointers, parameterSetSizes, 4, &videoFormatDescr); 
      [backend setVideoFormatDescr:videoFormatDescr]; 
      [backend setSearchForSPSAndPPS:false]; 
      NSLog(@"Found all data for CMVideoFormatDescription. Creation: %@.", (status == noErr) ? @"successfully." : @"failed."); 
     } 
    } 
    if (nalu_type == 1 || nalu_type == 5) { 
     CMBlockBufferRef videoBlock = NULL; 
     OSStatus status = CMBlockBufferCreateWithMemoryBlock(NULL, info.data, info.size, kCFAllocatorNull, NULL, 0, info.size, 0, &videoBlock); 
     NSLog(@"BlockBufferCreation: %@", (status == kCMBlockBufferNoErr) ? @"successfully." : @"failed."); 
     const uint8_t sourceBytes[] = {(uint8_t)(info.size >> 24), (uint8_t)(info.size >> 16), (uint8_t)(info.size >> 8), (uint8_t)info.size}; 
     status = CMBlockBufferReplaceDataBytes(sourceBytes, videoBlock, 0, 4); 
     NSLog(@"BlockBufferReplace: %@", (status == kCMBlockBufferNoErr) ? @"successfully." : @"failed."); 

     CMSampleBufferRef sbRef = NULL; 
     const size_t sampleSizeArray[] = {info.size}; 

     status = CMSampleBufferCreate(kCFAllocatorDefault, videoBlock, true, NULL, NULL, backend.videoFormatDescr, 1, 0, NULL, 1, sampleSizeArray, &sbRef); 
     NSLog(@"SampleBufferCreate: %@", (status == noErr) ? @"successfully." : @"failed."); 

     CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sbRef, YES); 
     CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); 
     CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue); 

     NSLog(@"Error: %@, Status:%@", backend.displayLayer.error, (backend.displayLayer.status == AVQueuedSampleBufferRenderingStatusUnknown)?@"unknown":((backend.displayLayer.status == AVQueuedSampleBufferRenderingStatusRendering)?@"rendering":@"failed")); 
     dispatch_async(dispatch_get_main_queue(),^{ 
      [backend.displayLayer enqueueSampleBuffer:sbRef]; 
      [backend.displayLayer setNeedsDisplay]; 
     }); 

    } 

    gst_memory_unmap(memory, &info); 
    gst_memory_unref(memory); 
    gst_buffer_unref(buffer); 

    return GST_FLOW_OK; 
} 

@implementation GStreamerBackend 

- (instancetype)init 
{ 
    if (self = [super init]) { 
     self.searchForSPSAndPPS = true; 
     self.ppsData = nil; 
     self.spsData = nil; 
     self.displayLayer = [[AVSampleBufferDisplayLayer alloc] init]; 
     self.displayLayer.bounds = CGRectMake(0, 0, 300, 300); 
     self.displayLayer.backgroundColor = [UIColor blackColor].CGColor; 
     self.displayLayer.position = CGPointMake(500, 500); 
     self.queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); 
     dispatch_async(self.queue, ^{ 
      [self app_function]; 
     }); 
    } 
    return self; 
} 

- (void)start 
{ 
    if(gst_element_set_state(self.pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { 
     NSLog(@"Failed to set pipeline to playing"); 
    } 
} 

- (void)app_function 
{ 
    GstElement *udpsrc, *rtphdepay, *capsfilter; 
    GMainContext *context; /* GLib context used to run the main loop */ 
    GMainLoop *main_loop; /* GLib main loop */ 


    context = g_main_context_new(); 
    g_main_context_push_thread_default(context); 

    g_set_application_name ("appsink"); 

    self.pipeline = gst_pipeline_new ("testpipe"); 

    udpsrc = gst_element_factory_make ("udpsrc", "udpsrc"); 
    GstCaps *caps = gst_caps_new_simple("application/x-rtp", "media", G_TYPE_STRING, "video", "clock-rate", G_TYPE_INT, 90000, "encoding-name", G_TYPE_STRING, "H264", NULL); 
    g_object_set(udpsrc, "caps", caps, "port", 5000, NULL); 
    gst_caps_unref(caps); 
    rtphdepay = gst_element_factory_make("rtph264depay", "rtph264depay"); 
    capsfilter = gst_element_factory_make("capsfilter", "capsfilter"); 
    caps = gst_caps_new_simple("video/x-h264", "streamformat", G_TYPE_STRING, "byte-stream", "alignment", G_TYPE_STRING, "nal", NULL); 
    g_object_set(capsfilter, "caps", caps, NULL); 
    self.appsink = gst_element_factory_make ("appsink", "appsink"); 

    gst_bin_add_many (GST_BIN (self.pipeline), udpsrc, rtphdepay, capsfilter, self.appsink, NULL); 

    if(!gst_element_link_many (udpsrc, rtphdepay, capsfilter, self.appsink, NULL)) { 
     NSLog(@"Cannot link gstreamer elements"); 
     exit (1); 
    } 

    if(gst_element_set_state(self.pipeline, GST_STATE_READY) != GST_STATE_CHANGE_SUCCESS) 
     NSLog(@"could not change to ready"); 

    GstAppSinkCallbacks callbacks = { NULL, NULL, new_sample, 
     NULL, NULL}; 
    gst_app_sink_set_callbacks (GST_APP_SINK(self.appsink), &callbacks, (__bridge gpointer)(self), NULL); 

    main_loop = g_main_loop_new (context, FALSE); 
    g_main_loop_run (main_loop); 


    /* Free resources */ 
    g_main_loop_unref (main_loop); 
    main_loop = NULL; 
    g_main_context_pop_thread_default(context); 
    g_main_context_unref (context); 
    gst_element_set_state (GST_ELEMENT (self.pipeline), GST_STATE_NULL); 
    gst_object_unref (GST_OBJECT (self.pipeline)); 
} 

@end 

앱을 실행하고 아이폰 OS 장치로 스트리밍하기 시작했을 때 나는 무엇을 얻을 :

NALU with Type "Sequence parameter set (non-VCL)" received. 
NALU with Type "Picture parameter set (non-VCL)" received. 

Found all data for CMVideoFormatDescription. Creation: successfully.. 

NALU with Type "Coded slice of an IDR picture (VCL)" received. 
BlockBufferCreation: successfully. 
BlockBufferReplace: successfully. 
SampleBufferCreate: successfully. 
Error: (null), Status:unknown 

NALU with Type "Coded slice of a non-IDR picture (VCL)" received. 
BlockBufferCreation: successfully. 
BlockBufferReplace: successfully. 
SampleBufferCreate: successfully. 
Error: (null), Status:rendering 
[...] (repetition of the last 5 lines) 

이 그래서가 보인다 디코딩해야하지만 내 문제는 AVSampleBufferDisplayLayer에서 아무 것도 볼 수 없다는 것입니다. kCMSampleAttachmentKey_DisplayImmediately에 문제가있을 수 있지만 here (see the 'important' note)에게 말한 것처럼 설정했습니다.

모든 아이디어는 환영합니다;)

Answer 1

지금 사용 가능합니다. 각 NALU의 길이는 길이 헤더 자체를 포함하지 않습니다. 그래서 내 sourceBytes 그것을 사용하기 전에 내 info.size 4 빼기 할.

Answer 2

귀하의 코드로 지시하면, AVSampleBufferDisplayLayer를 사용하여 라이브 H.264 스트림을 디코딩하고 표시하는 프로그램을 작성합니다 .H.264 NAL 단위를 수신하려면 live555 대신 GSStream을 사용하십시오.

불행히도 내 앱은 몇 프레임 만 표시하고 더 이상 표시 할 수있는 이미지가 없습니다. 동일한 문제를 만난 적이 있습니까?