Detailed explanation of scrcpy client code walk through H264 raw stream decoding process

This chapter is mainly about smoothing scrcpy client Decoding process , Sort out according to the following process ：

1>. decoder_init(&s->decoder); ///> 5. decoder_init();
2>. stream_init(&s->stream, s->server.video_socket, &stream_cbs, NULL);
3>. stream_add_sink(&s->stream, &dec->packet_sink); ///> 8. stream_add_sink();

4>. screen_init(&s->screen, &screen_params); ///> 12. screen_init();
5>. decoder_add_sink(&s->decoder, &s->screen.frame_sink); ///> 13. decoder_add_sink();
6>. stream_start(&s->stream); ///> 14+. Stream start configuration

The first 1 Step Decoder initialization content

void decoder_init(struct decoder *decoder) {
    
    decoder->sink_count = 0;

    static const struct sc_packet_sink_ops ops = {
    
        .open = decoder_packet_sink_open,
        .close = decoder_packet_sink_close,
        .push = decoder_packet_sink_push,
    };
    decoder->packet_sink.ops = &ops;
}
///>  Here is the source code initialization  decoder->packet_sink.ops  Structural content , The program is in  run_stream()  In the thread , Just call directly  sink->ops->push()  function 
///>  Feed the data stream to the encoder , Have a look first  push  function .
static bool
decoder_packet_sink_push(struct sc_packet_sink *sink, const AVPacket *packet) {
    
    struct decoder *decoder = DOWNCAST(sink);
    return decoder_push(decoder, packet);
}
static bool
decoder_push(struct decoder *decoder, const AVPacket *packet) {
    
    bool is_config = packet->pts == AV_NOPTS_VALUE;
    if (is_config) {
    
        // nothing to do
        return true;
    }

    int ret = avcodec_send_packet(decoder->codec_ctx, packet);
    if (ret < 0 && ret != AVERROR(EAGAIN)) {
    
        LOGE("Could not send video packet: %d", ret);
        return false;
    }
    ret = avcodec_receive_frame(decoder->codec_ctx, decoder->frame);
    if (!ret) {
    
        // a frame was received
        bool ok = push_frame_to_sinks(decoder, decoder->frame);
        // A frame lost should not make the whole pipeline fail. The error, if
        // any, is already logged.
        (void) ok;

        av_frame_unref(decoder->frame);
    } else if (ret != AVERROR(EAGAIN)) {
    
        LOGE("Could not receive video frame: %d", ret);
        return false;
    }
    return true;
}
///>  Here the final call is  decoder_push  function , From the function implementation, we can see that the call  ffmpeg  Interface ,avcodec_send_packet  Function to send data ;
///>  call  avcodec_receive_frame()  Function to get the decoded frame data , And put  frame  Deliver to  sink  entrance .

The first 2 Step Initialize the flow part

    static const struct stream_callbacks stream_cbs = {
    
        .on_eos = stream_on_eos,
    };
    stream_init(&s->stream, s->server.video_socket, &stream_cbs, NULL);
    void
///>  Implementation of function 
stream_init(struct stream *stream, socket_t socket,
            const struct stream_callbacks *cbs, void *cbs_userdata) {
    
    stream->socket = socket;
    stream->pending = NULL;
    stream->sink_count = 0;

    assert(cbs && cbs->on_eos);

    stream->cbs = cbs;
    stream->cbs_userdata = cbs_userdata;
}
///>  Stream return function  stream_on_eos  Function implementation content 
static void
stream_on_eos(struct stream *stream, void *userdata) {
    
    (void) stream;
    (void) userdata;

    SDL_Event stop_event;
    stop_event.type = EVENT_STREAM_STOPPED;
    SDL_PushEvent(&stop_event);
}
///>  Sent here  sdl event  to  event_loop()  function , The decoding of the display stream can be stopped in this way .
static bool
event_loop(struct scrcpy *s, const struct scrcpy_options *options) {
    
    SDL_Event event;
    while (SDL_WaitEvent(&event)) {
    
        enum event_result result = handle_event(s, options, &event);
        switch (result) {
    
            case EVENT_RESULT_STOPPED_BY_USER:
                return true;
            case EVENT_RESULT_STOPPED_BY_EOS:
                LOGW("Device disconnected");
                return false;
            case EVENT_RESULT_CONTINUE:
                break;
        }
    }
    return false;
}

The first 3 Step hold The decoder slot is added to the stream

void
stream_add_sink(struct stream *stream, struct sc_packet_sink *sink) {
    
    assert(stream->sink_count < STREAM_MAX_SINKS);
    assert(sink);
    assert(sink->ops);
    stream->sinks[stream->sink_count++] = sink;
}
///>  have a look  sc_packet_sink  Structure definition 
struct sc_packet_sink {
    
    const struct sc_packet_sink_ops *ops;
};

struct sc_packet_sink_ops {
    
    bool (*open)(struct sc_packet_sink *sink, const AVCodec *codec);
    void (*close)(struct sc_packet_sink *sink);
    bool (*push)(struct sc_packet_sink *sink, const AVPacket *packet);
};
///>  stay   The first 1 Step   Has been described in , Temporarily define the slot as ： With open 、 Turn off the function , And you can put  AVPacket  Put the package data in .
///>  We can call it   Slot .

The first 4 Step Initialize local screen

Simplify the source code , Keep relevant content .

bool screen_init(struct screen *screen, const struct screen_params *params) 
{
    
    screen->resize_pending = false;
    screen->has_frame = false;
    screen->fullscreen = false;
    screen->maximized = false;

    static const struct sc_video_buffer_callbacks cbs = {
    
        .on_new_frame = sc_video_buffer_on_new_frame,
    };

    ok = sc_video_buffer_start(&screen->vb);			///>  This function creates  run_buffering()  Threads 
    if (!ok) {
    
        LOGE("Could not start video_buffer");
        goto error_destroy_video_buffer;
    }


    static const struct sc_frame_sink_ops ops = {
    
        .open = screen_frame_sink_open,
        .close = screen_frame_sink_close,
        .push = screen_frame_sink_push,
    };

    screen->frame_sink.ops = &ops;
}
///>  This function   establish  run_buffering  Threads ,  And set up  screen->frame_sink.ops  Structural content 

The first 5 Step Add the screen slot to the decoder

adopt decoder_add_sink(&s->decoder, &s->screen.frame_sink) Function screen.frame_sink Add to decoder ,

void
decoder_add_sink(struct decoder *decoder, struct sc_frame_sink *sink) {
    
    assert(decoder->sink_count < DECODER_MAX_SINKS);
    assert(sink);
    assert(sink->ops);
    decoder->sinks[decoder->sink_count++] = sink;
}

stay The first 4 Step Described in , Take it off and let's have a look screen_frame_sink_push() Function implementation function .

bool
sc_video_buffer_push(struct sc_video_buffer *vb, const AVFrame *frame) {
    
    if (!vb->buffering_time) {
    
        // No buffering
        return sc_video_buffer_offer(vb, frame);
    }

    sc_mutex_lock(&vb->b.mutex);

    sc_tick pts = SC_TICK_FROM_US(frame->pts);
    sc_clock_update(&vb->b.clock, sc_tick_now(), pts);
    sc_cond_signal(&vb->b.wait_cond);

    if (vb->b.clock.count == 1) {
    
        sc_mutex_unlock(&vb->b.mutex);
        // First frame, offer it immediately, for two reasons:
        // - not to delay the opening of the scrcpy window
        // - the buffering estimation needs at least two clock points, so it
        // could not handle the first frame
        return sc_video_buffer_offer(vb, frame);
    }

    struct sc_video_buffer_frame *vb_frame = sc_video_buffer_frame_new(frame);
    if (!vb_frame) {
    
        sc_mutex_unlock(&vb->b.mutex);
        LOGE("Could not allocate frame");
        return false;
    }

#ifndef SC_BUFFERING_NDEBUG
    vb_frame->push_date = sc_tick_now();
#endif
    sc_queue_push(&vb->b.queue, next, vb_frame);
    sc_cond_signal(&vb->b.queue_cond);
    sc_mutex_unlock(&vb->b.mutex);
    return true;
}
///>  in other words  decoder->sinks->push()  function , Is to execute this function .

The first 6 Step Stream start configuration

bool stream_start(struct stream *stream) 
{
    
    LOGD("Starting stream thread");
    printf("%s, %d DEBUG\n", __FILE__, __LINE__);
    bool ok = sc_thread_create(&stream->thread, run_stream, "stream", stream);
    if (!ok) {
    
        LOGC("Could not start stream thread");
        return false;
    }
    return true;
}
///>  Create... Here  run_stream  Threads , The contents are as follows ：

static int run_stream(void *data) 
{
    
	struct stream *stream = data;
	AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264);
	stream->codec_ctx = avcodec_alloc_context3(codec);

	stream_open_sinks(stream, codec);

	stream->parser = av_parser_init(AV_CODEC_ID_H264);
	AVPacket *packet = av_packet_alloc();


    for (;;) {
    
        bool ok = stream_recv_packet(stream, packet);
        if (!ok) {
    
            // end of stream
            break;
        }

        ok = stream_push_packet(stream, packet);
        av_packet_unref(packet);
        if (!ok) {
    
            // cannot process packet (error already logged)
            break;
        }
    }

    return 0;
}
///>  This function is a simplified version of related functions , You can see initialization   Decoder parameters and initialize the decoder 
///>  Turn on the flow  sink  The switch of , Can be directed to  sink  in  push  Stream data .
///<  In circulation  stream_recv_packet()  Read network socket The data of ,stream_push_packet() Function call No 1 In step 
///  Description of the  decoder_push()  function , The data is decoded ,push_frame_to_sinks(decoder, decoder->frame) function 
///  Execution is  sc_video_buffer_push() function , That is, the data is sent to  screen  In the object , How to get the data in this object ？

The first 7 Step run_buffering Threads

stay The first 4 Create when you step run_buffering Threads , Let's see what the thread is doing .

static int
run_buffering(void *data) {
    
    struct sc_video_buffer *vb = data;

    assert(vb->buffering_time > 0);

    for (;;) {
    
        sc_mutex_lock(&vb->b.mutex);

        while (!vb->b.stopped && sc_queue_is_empty(&vb->b.queue)) {
    
            sc_cond_wait(&vb->b.queue_cond, &vb->b.mutex);
        }

        if (vb->b.stopped) {
    
            sc_mutex_unlock(&vb->b.mutex);
            goto stopped;
        }

        struct sc_video_buffer_frame *vb_frame;
        sc_queue_take(&vb->b.queue, next, &vb_frame);     ///>  take  frame  data 

        sc_tick max_deadline = sc_tick_now() + vb->buffering_time;
        // PTS (written by the server) are expressed in microseconds
        sc_tick pts = SC_TICK_TO_US(vb_frame->frame->pts);

        bool timed_out = false;
        while (!vb->b.stopped && !timed_out) {
    
            sc_tick deadline = sc_clock_to_system_time(&vb->b.clock, pts)
                             + vb->buffering_time;
            if (deadline > max_deadline) {
    
                deadline = max_deadline;
            }

            timed_out =
                !sc_cond_timedwait(&vb->b.wait_cond, &vb->b.mutex, deadline);
        }

        if (vb->b.stopped) {
    
            sc_video_buffer_frame_delete(vb_frame);
            sc_mutex_unlock(&vb->b.mutex);
            goto stopped;
        }

        sc_mutex_unlock(&vb->b.mutex);

#ifndef SC_BUFFERING_NDEBUG
        LOGD("Buffering: %" PRItick ";%" PRItick ";%" PRItick,
             pts, vb_frame->push_date, sc_tick_now());
#endif

        sc_video_buffer_offer(vb, vb_frame->frame);      ///>  hold frame data push To the renderer 

        sc_video_buffer_frame_delete(vb_frame);
    }

stopped:
    // Flush queue
    while (!sc_queue_is_empty(&vb->b.queue)) {
    
        struct sc_video_buffer_frame *vb_frame;
        sc_queue_take(&vb->b.queue, next, &vb_frame);
        sc_video_buffer_frame_delete(vb_frame);
    }

    LOGD("Buffering thread ended");

    return 0;
}

This section describes "stream -> decoder -> render " The process ,H264 Detailed process of raw stream decoding .