vulkan_decode.c

Go to the documentation of this file.

/*
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */
 
#include "libavutil/refstruct.h"
#include "vulkan_video.h"
#include "vulkan_decode.h"
#include "config_components.h"
#include "libavutil/avassert.h"
#include "libavutil/mem.h"
#include "libavutil/vulkan_loader.h"
 
#define DECODER_IS_SDR(codec_id) \
    (((codec_id) == AV_CODEC_ID_FFV1) || \
     ((codec_id) == AV_CODEC_ID_DPX) || \
     ((codec_id) == AV_CODEC_ID_PRORES_RAW) || \
     ((codec_id) == AV_CODEC_ID_PRORES))
 
#if CONFIG_H264_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc;
#endif
#if CONFIG_HEVC_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_hevc_desc;
#endif
#if CONFIG_VP9_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_vp9_desc;
#endif
#if CONFIG_AV1_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_av1_desc;
#endif
#if CONFIG_FFV1_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_ffv1_desc;
#endif
#if CONFIG_PRORES_RAW_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_prores_raw_desc;
#endif
#if CONFIG_PRORES_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_prores_desc;
#endif
#if CONFIG_DPX_VULKAN_HWACCEL
extern const FFVulkanDecodeDescriptor ff_vk_dec_dpx_desc;
#endif
 
static const FFVulkanDecodeDescriptor *dec_descs[] = {
#if CONFIG_H264_VULKAN_HWACCEL
    &ff_vk_dec_h264_desc,
#endif
#if CONFIG_HEVC_VULKAN_HWACCEL
    &ff_vk_dec_hevc_desc,
#endif
#if CONFIG_VP9_VULKAN_HWACCEL
    &ff_vk_dec_vp9_desc,
#endif
#if CONFIG_AV1_VULKAN_HWACCEL
    &ff_vk_dec_av1_desc,
#endif
#if CONFIG_FFV1_VULKAN_HWACCEL
    &ff_vk_dec_ffv1_desc,
#endif
#if CONFIG_PRORES_RAW_VULKAN_HWACCEL
    &ff_vk_dec_prores_raw_desc,
#endif
#if CONFIG_PRORES_VULKAN_HWACCEL
    &ff_vk_dec_prores_desc,
#endif
#if CONFIG_DPX_VULKAN_HWACCEL
    &ff_vk_dec_dpx_desc,
#endif
};
 
typedef struct FFVulkanDecodeProfileData {
     VkVideoDecodeH264ProfileInfoKHR h264_profile;
     VkVideoDecodeH265ProfileInfoKHR h265_profile;
#if CONFIG_VP9_VULKAN_HWACCEL
     VkVideoDecodeVP9ProfileInfoKHR vp9_profile;
#endif
     VkVideoDecodeAV1ProfileInfoKHR av1_profile;
 
    VkVideoDecodeUsageInfoKHR usage;
    VkVideoProfileInfoKHR profile;
    VkVideoProfileListInfoKHR profile_list;
} FFVulkanDecodeProfileData;
 
static const FFVulkanDecodeDescriptor *get_codecdesc(enum AVCodecID codec_id)
{
    for (size_t i = 0; i < FF_ARRAY_ELEMS(dec_descs); i++)
        if (dec_descs[i]->codec_id == codec_id)
            return dec_descs[i];
    av_assert1(!"no codec descriptor");
    return NULL;
}
 
static const VkVideoProfileInfoKHR *get_video_profile(FFVulkanDecodeShared *ctx, enum AVCodecID codec_id)
{
    const VkVideoProfileListInfoKHR *profile_list;
 
    VkStructureType profile_struct_type =
        codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :
        codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :
#if CONFIG_VP9_VULKAN_HWACCEL
        codec_id == AV_CODEC_ID_VP9  ? VK_STRUCTURE_TYPE_VIDEO_DECODE_VP9_PROFILE_INFO_KHR :
#endif
        codec_id == AV_CODEC_ID_AV1  ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :
                                       VK_STRUCTURE_TYPE_MAX_ENUM;
    if (profile_struct_type == VK_STRUCTURE_TYPE_MAX_ENUM)
        return NULL;
 
    profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,
                                     VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
    if (!profile_list)
        return NULL;
 
    for (int i = 0; i < profile_list->profileCount; i++)
        if (ff_vk_find_struct(profile_list->pProfiles[i].pNext, profile_struct_type))
            return &profile_list->pProfiles[i];
 
    return NULL;
}
 
int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
{
    int err;
    FFVulkanDecodeContext *src_ctx = src->internal->hwaccel_priv_data;
    FFVulkanDecodeContext *dst_ctx = dst->internal->hwaccel_priv_data;
 
    av_refstruct_replace(&dst_ctx->shared_ctx, src_ctx->shared_ctx);
 
    err = av_buffer_replace(&dst_ctx->session_params, src_ctx->session_params);
    if (err < 0)
        return err;
 
    dst_ctx->dedicated_dpb = src_ctx->dedicated_dpb;
    dst_ctx->external_fg = src_ctx->external_fg;
 
    return 0;
}
 
int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
{
    FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
    av_buffer_unref(&dec->session_params);
    return 0;
}
 
static AVFrame *vk_get_dpb_pool(FFVulkanDecodeShared *ctx)
{
    int err;
    AVFrame *avf = av_frame_alloc();
    if (!avf)
        return NULL;
 
    err = av_hwframe_get_buffer(ctx->common.dpb_hwfc_ref, avf, 0x0);
    if (err < 0)
        av_frame_free(&avf);
 
    return avf;
}
 
static void init_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vkpic)
{
    FFVulkanDecodeShared *ctx = dec->shared_ctx;
    FFVulkanFunctions *vk = &ctx->s.vkfn;
 
    vkpic->dpb_frame     = NULL;
    for (int i = 0; i < AV_NUM_DATA_POINTERS; i++) {
        vkpic->view.ref[i]  = VK_NULL_HANDLE;
        vkpic->view.out[i]  = VK_NULL_HANDLE;
        vkpic->view.dst[i]  = VK_NULL_HANDLE;
    }
 
    vkpic->destroy_image_view = vk->DestroyImageView;
    vkpic->wait_semaphores = vk->WaitSemaphores;
    vkpic->invalidate_memory_ranges = vk->InvalidateMappedMemoryRanges;
}
 
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,
                               FFVulkanDecodePicture *vkpic, int is_current,
                               int alloc_dpb)
{
    int err;
    FFVulkanDecodeShared *ctx = dec->shared_ctx;
 
    vkpic->slices_size = 0;
 
    /* If the decoder made a blank frame to make up for a missing ref, or the
     * frame is the current frame so it's missing one, create a re-representation */
    if (vkpic->view.ref[0])
        return 0;
 
    init_frame(dec, vkpic);
 
    if (ctx->common.layered_dpb && alloc_dpb) {
        vkpic->view.ref[0] = ctx->common.layered_view;
        vkpic->view.aspect_ref[0] = ctx->common.layered_aspect;
    } else if (alloc_dpb) {
        AVHWFramesContext *dpb_frames = (AVHWFramesContext *)ctx->common.dpb_hwfc_ref->data;
        AVVulkanFramesContext *dpb_hwfc = dpb_frames->hwctx;
 
        vkpic->dpb_frame = vk_get_dpb_pool(ctx);
        if (!vkpic->dpb_frame)
            return AVERROR(ENOMEM);
 
        err = ff_vk_create_view(&ctx->s, &ctx->common,
                                &vkpic->view.ref[0], &vkpic->view.aspect_ref[0],
                                (AVVkFrame *)vkpic->dpb_frame->data[0],
                                dpb_hwfc->format[0], VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR);
        if (err < 0)
            return err;
 
        vkpic->view.dst[0] = vkpic->view.ref[0];
    }
 
    if (!alloc_dpb || is_current) {
        AVHWFramesContext *frames = (AVHWFramesContext *)pic->hw_frames_ctx->data;
        AVVulkanFramesContext *hwfc = frames->hwctx;
 
        err = ff_vk_create_view(&ctx->s, &ctx->common,
                                &vkpic->view.out[0], &vkpic->view.aspect[0],
                                (AVVkFrame *)pic->data[0],
                                hwfc->format[0],
                                VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
                                (!is_current ? VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR : 0));
        if (err < 0)
            return err;
 
        if (!alloc_dpb) {
            vkpic->view.ref[0] = vkpic->view.out[0];
            vkpic->view.aspect_ref[0] = vkpic->view.aspect[0];
        }
    }
 
    return 0;
}
 
int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic,
                                   FFVulkanDecodePicture *vkpic, int is_current,
                                   enum FFVkShaderRepFormat rep_fmt, int alloc_dpb)
{
    int err;
    FFVulkanDecodeShared *ctx = dec->shared_ctx;
    AVHWFramesContext *frames = (AVHWFramesContext *)pic->hw_frames_ctx->data;
 
    vkpic->slices_size = 0;
 
    if (vkpic->view.ref[0])
        return 0;
 
    init_frame(dec, vkpic);
 
    for (int i = 0; i < av_pix_fmt_count_planes(frames->sw_format); i++) {
        if (alloc_dpb) {
            vkpic->dpb_frame = vk_get_dpb_pool(ctx);
            if (!vkpic->dpb_frame)
                return AVERROR(ENOMEM);
 
            err = ff_vk_create_imageview(&ctx->s,
                                         &vkpic->view.ref[i], &vkpic->view.aspect_ref[i],
                                         vkpic->dpb_frame, i, rep_fmt);
            if (err < 0)
                return err;
 
            vkpic->view.dst[i] = vkpic->view.ref[i];
        }
 
        if (!alloc_dpb || is_current) {
            err = ff_vk_create_imageview(&ctx->s,
                                         &vkpic->view.out[i], &vkpic->view.aspect[i],
                                         pic, i, rep_fmt);
            if (err < 0)
                return err;
 
            if (!alloc_dpb) {
                vkpic->view.ref[i] = vkpic->view.out[i];
                vkpic->view.aspect_ref[i] = vkpic->view.aspect[i];
            }
        }
    }
 
    return 0;
}
 
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp,
                           const uint8_t *data, size_t size, int add_startcode,
                           uint32_t *nb_slices, const uint32_t **offsets)
{
    FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
    FFVulkanDecodeShared *ctx = dec->shared_ctx;
 
    static const uint8_t startcode_prefix[3] = { 0x0, 0x0, 0x1 };
    const size_t startcode_len = add_startcode ? sizeof(startcode_prefix) : 0;
    const int nb = nb_slices ? *nb_slices : 0;
    uint8_t *slices;
    uint32_t *slice_off;
    FFVkBuffer *vkbuf;
 
    size_t new_size = vp->slices_size + startcode_len + size +
                      ctx->caps.minBitstreamBufferSizeAlignment;
    new_size = FFALIGN(new_size, ctx->caps.minBitstreamBufferSizeAlignment);
 
    if (offsets) {
        slice_off = av_fast_realloc(dec->slice_off, &dec->slice_off_max,
                                    (nb + 1)*sizeof(slice_off));
        if (!slice_off)
            return AVERROR(ENOMEM);
 
        *offsets = dec->slice_off = slice_off;
 
        slice_off[nb] = vp->slices_size;
    }
 
    vkbuf = vp->slices_buf ? (FFVkBuffer *)vp->slices_buf->data : NULL;
    if (!vkbuf || vkbuf->size < new_size) {
        int err;
        AVBufferRef *new_ref;
        FFVkBuffer *new_buf;
 
        /* No point in requesting anything smaller. */
        size_t buf_size = FFMAX(new_size, 1024*1024);
 
        /* Align buffer to nearest power of two. Makes fragmentation management
         * easier, and gives us ample headroom. */
        buf_size = 2 << av_log2(buf_size);
 
        err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_ref,
                                      DECODER_IS_SDR(avctx->codec_id) ?
                                      (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
                                       VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) :
                                      VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR,
                                      ctx->s.hwfc->create_pnext, buf_size,
                                      VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
                                      (DECODER_IS_SDR(avctx->codec_id) ?
                                       VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : 0x0));
        if (err < 0)
            return err;
 
        new_buf = (FFVkBuffer *)new_ref->data;
 
        /* Copy data from the old buffer */
        if (vkbuf) {
            memcpy(new_buf->mapped_mem, vkbuf->mapped_mem, vp->slices_size);
            av_buffer_unref(&vp->slices_buf);
        }
 
        vp->slices_buf = new_ref;
        vkbuf = new_buf;
    }
    slices = vkbuf->mapped_mem;
 
    /* Startcode */
    memcpy(slices + vp->slices_size, startcode_prefix, startcode_len);
 
    /* Slice data */
    memcpy(slices + vp->slices_size + startcode_len, data, size);
 
    if (nb_slices)
        *nb_slices = nb + 1;
 
    vp->slices_size += startcode_len + size;
 
    return 0;
}
 
static int create_empty_session_parameters(AVCodecContext *avctx,
                                           FFVulkanDecodeShared *ctx,
                                           VkVideoSessionParametersKHR *empty_session_params)
{
    if (avctx->codec_id == AV_CODEC_ID_VP9)
        return 0;
 
    VkVideoDecodeH264SessionParametersCreateInfoKHR h264_params = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_CREATE_INFO_KHR,
    };
    VkVideoDecodeH265SessionParametersCreateInfoKHR h265_params = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_SESSION_PARAMETERS_CREATE_INFO_KHR,
    };
    StdVideoAV1SequenceHeader av1_empty_seq = { 0 };
    VkVideoDecodeAV1SessionParametersCreateInfoKHR av1_params = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_CREATE_INFO_KHR,
        .pStdSequenceHeader = &av1_empty_seq,
    };
    VkVideoSessionParametersCreateInfoKHR session_params_create = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
        .pNext = avctx->codec_id == AV_CODEC_ID_H264 ? (void *)&h264_params :
                 avctx->codec_id == AV_CODEC_ID_HEVC ? (void *)&h265_params :
                 avctx->codec_id == AV_CODEC_ID_AV1  ? (void *)&av1_params  :
                 NULL,
        .videoSession = ctx->common.session,
    };
 
    VkResult ret;
    FFVulkanContext *s = &ctx->s;
    FFVulkanFunctions *vk = &s->vkfn;
    ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
                                              s->hwctx->alloc, empty_session_params);
    if (ret != VK_SUCCESS) {
        av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
               ff_vk_ret2str(ret));
        return AVERROR_EXTERNAL;
    }
 
    return 0;
}
 
static int decode_reset(AVCodecContext *avctx, FFVulkanDecodeShared *ctx)
{
    int err;
    FFVulkanContext *s = &ctx->s;
    FFVulkanFunctions *vk = &s->vkfn;
 
    VkVideoBeginCodingInfoKHR decode_start = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
        .videoSession = ctx->common.session,
    };
 
    if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_2)) {
        err = create_empty_session_parameters(avctx, ctx,
                                              &decode_start.videoSessionParameters);
        if (err < 0)
            return err;
    }
 
    VkVideoCodingControlInfoKHR decode_ctrl = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_CODING_CONTROL_INFO_KHR,
        .flags = VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR,
    };
    VkVideoEndCodingInfoKHR decode_end = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
    };
 
    FFVkExecContext *exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
    ff_vk_exec_start(&ctx->s, exec);
 
    vk->CmdBeginVideoCodingKHR(exec->buf, &decode_start);
    vk->CmdControlVideoCodingKHR(exec->buf, &decode_ctrl);
    vk->CmdEndVideoCodingKHR(exec->buf, &decode_end);
 
    err = ff_vk_exec_submit(&ctx->s, exec);
 
    if (decode_start.videoSessionParameters) {
        /* Wait to complete to delete the temporary session parameters */
        if (err >= 0)
            ff_vk_exec_wait(&ctx->s, exec);
 
        vk->DestroyVideoSessionParametersKHR(s->hwctx->act_dev,
                                             decode_start.videoSessionParameters,
                                             s->hwctx->alloc);
    }
 
    if (err < 0)
        av_log(avctx, AV_LOG_ERROR, "Unable to reset decoder: %s",
               ff_vk_ret2str(err));
 
    return err;
}
 
int ff_vk_decode_frame(AVCodecContext *avctx,
                       AVFrame *pic,    FFVulkanDecodePicture *vp,
                       AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
{
    int err;
    VkResult ret;
    VkCommandBuffer cmd_buf;
    FFVkBuffer *sd_buf;
 
    FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
    FFVulkanDecodeShared *ctx = dec->shared_ctx;
    FFVulkanFunctions *vk = &ctx->s.vkfn;
 
    /* Output */
    AVVkFrame *vkf = (AVVkFrame *)pic->buf[0]->data;
 
    /* Quirks */
    const int layered_dpb = ctx->common.layered_dpb;
 
    VkVideoBeginCodingInfoKHR decode_start = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
        .videoSession = ctx->common.session,
        .videoSessionParameters = dec->session_params ?
                                  *((VkVideoSessionParametersKHR *)dec->session_params->data) :
                                  VK_NULL_HANDLE,
        .referenceSlotCount = vp->decode_info.referenceSlotCount,
        .pReferenceSlots = vp->decode_info.pReferenceSlots,
    };
    VkVideoEndCodingInfoKHR decode_end = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
    };
 
    VkImageMemoryBarrier2 img_bar[37];
    int nb_img_bar = 0;
    size_t data_size = FFALIGN(vp->slices_size,
                               ctx->caps.minBitstreamBufferSizeAlignment);
 
    FFVkExecContext *exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
 
    /* The current decoding reference has to be bound as an inactive reference */
    VkVideoReferenceSlotInfoKHR *cur_vk_ref;
    cur_vk_ref = (void *)&decode_start.pReferenceSlots[decode_start.referenceSlotCount];
    cur_vk_ref[0] = vp->ref_slot;
    cur_vk_ref[0].slotIndex = -1;
    decode_start.referenceSlotCount++;
 
    sd_buf = (FFVkBuffer *)vp->slices_buf->data;
 
    /* Flush if needed */
    if (!(sd_buf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
        VkMappedMemoryRange flush_buf = {
            .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
            .memory = sd_buf->mem,
            .offset = 0,
            .size = FFALIGN(vp->slices_size,
                            ctx->s.props.properties.limits.nonCoherentAtomSize),
        };
 
        ret = vk->FlushMappedMemoryRanges(ctx->s.hwctx->act_dev, 1, &flush_buf);
        if (ret != VK_SUCCESS) {
            av_log(avctx, AV_LOG_ERROR, "Failed to flush memory: %s\n",
                   ff_vk_ret2str(ret));
            return AVERROR_EXTERNAL;
        }
    }
 
    vp->decode_info.srcBuffer       = sd_buf->buf;
    vp->decode_info.srcBufferOffset = 0;
    vp->decode_info.srcBufferRange  = data_size;
 
    /* Start command buffer recording */
    err = ff_vk_exec_start(&ctx->s, exec);
    if (err < 0)
        return err;
    cmd_buf = exec->buf;
 
    /* Slices */
    err = ff_vk_exec_add_dep_buf(&ctx->s, exec, &vp->slices_buf, 1, 0);
    if (err < 0)
        return err;
    vp->slices_buf = NULL; /* Owned by the exec buffer from now on */
 
    /* Parameters */
    err = ff_vk_exec_add_dep_buf(&ctx->s, exec, &dec->session_params, 1, 1);
    if (err < 0)
        return err;
 
    err = ff_vk_exec_add_dep_frame(&ctx->s, exec, pic,
                                   VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
                                   VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
    if (err < 0)
        return err;
 
    err = ff_vk_exec_mirror_sem_value(&ctx->s, exec, &vp->sem, &vp->sem_value,
                                      pic);
    if (err < 0)
        return err;
 
    /* Output image - change layout, as it comes from a pool */
    img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
        .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
        .pNext = NULL,
        .srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
        .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
        .srcAccessMask = VK_ACCESS_2_NONE,
        .dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
        .oldLayout = vkf->layout[0],
        .newLayout = (layered_dpb || vp->dpb_frame) ?
                     VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR :
                     VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, /* Spec, 07252 utter madness */
        .srcQueueFamilyIndex = vkf->queue_family[0],
        .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
        .image = vkf->img[0],
        .subresourceRange = (VkImageSubresourceRange) {
            .aspectMask = vp->view.aspect[0],
            .layerCount = 1,
            .levelCount = 1,
        },
    };
    ff_vk_exec_update_frame(&ctx->s, exec, pic,
                            &img_bar[nb_img_bar], &nb_img_bar);
 
    /* Reference for the current image, if existing and not layered */
    if (vp->dpb_frame) {
        err = ff_vk_exec_add_dep_frame(&ctx->s, exec, vp->dpb_frame,
                                       VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
                                       VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
        if (err < 0)
            return err;
    }
 
    if (!layered_dpb) {
        /* All references (apart from the current) for non-layered refs */
 
        for (int i = 0; i < vp->decode_info.referenceSlotCount; i++) {
            AVFrame *ref_frame = rpic[i];
            FFVulkanDecodePicture *rvp = rvkp[i];
            AVFrame *ref = rvp->dpb_frame ? rvp->dpb_frame : ref_frame;
 
            err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ref,
                                           VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
                                           VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
            if (err < 0)
                return err;
 
            if (err == 0) {
                err = ff_vk_exec_mirror_sem_value(&ctx->s, exec,
                                                  &rvp->sem, &rvp->sem_value,
                                                  ref);
                if (err < 0)
                    return err;
            }
 
            if (!rvp->dpb_frame) {
                AVVkFrame *rvkf = (AVVkFrame *)ref->data[0];
 
                img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
                    .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
                    .pNext = NULL,
                    .srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
                    .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
                    .srcAccessMask = VK_ACCESS_2_NONE,
                    .dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_READ_BIT_KHR |
                                     VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
                    .oldLayout = rvkf->layout[0],
                    .newLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR,
                    .srcQueueFamilyIndex = rvkf->queue_family[0],
                    .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
                    .image = rvkf->img[0],
                    .subresourceRange = (VkImageSubresourceRange) {
                        .aspectMask = rvp->view.aspect_ref[0],
                        .layerCount = 1,
                        .levelCount = 1,
                    },
                };
                ff_vk_exec_update_frame(&ctx->s, exec, ref,
                                        &img_bar[nb_img_bar], &nb_img_bar);
            }
        }
    } else if (vp->decode_info.referenceSlotCount ||
               vp->view.out[0] != vp->view.ref[0]) {
        /* Single barrier for a single layered ref */
        err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ctx->common.layered_frame,
                                       VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
                                       VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
        if (err < 0)
            return err;
    }
 
    /* Change image layout */
    vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
            .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
            .dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT,
            .pImageMemoryBarriers = img_bar,
            .imageMemoryBarrierCount = nb_img_bar,
        });
 
    /* Start, use parameters, decode and end decoding */
    vk->CmdBeginVideoCodingKHR(cmd_buf, &decode_start);
    vk->CmdDecodeVideoKHR(cmd_buf, &vp->decode_info);
    vk->CmdEndVideoCodingKHR(cmd_buf, &decode_end);
 
    /* End recording and submit for execution */
    return ff_vk_exec_submit(&ctx->s, exec);
}
 
void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp)
{
    AVVulkanDeviceContext *hwctx = dev_ctx->hwctx;
 
    VkSemaphoreWaitInfo sem_wait = (VkSemaphoreWaitInfo) {
        .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
        .pSemaphores = &vp->sem,
        .pValues = &vp->sem_value,
        .semaphoreCount = 1,
    };
 
    /* We do not have to lock the frame here because we're not interested
     * in the actual current semaphore value, but only that it's later than
     * the time we submitted the image for decoding. */
    if (vp->sem)
        vp->wait_semaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
 
    /* Free slices data */
    av_buffer_unref(&vp->slices_buf);
 
    /* Destroy image view (out) */
    for (int i = 0; i < AV_NUM_DATA_POINTERS; i++) {
        if (vp->view.out[i] && vp->view.out[i] != vp->view.dst[i])
            vp->destroy_image_view(hwctx->act_dev, vp->view.out[i], hwctx->alloc);
 
        /* Destroy image view (ref, unlayered) */
        if (vp->view.dst[i])
            vp->destroy_image_view(hwctx->act_dev, vp->view.dst[i], hwctx->alloc);
    }
 
    av_frame_free(&vp->dpb_frame);
}
 
static void free_common(AVRefStructOpaque unused, void *obj)
{
    FFVulkanDecodeShared *ctx = obj;
    FFVulkanContext *s = &ctx->s;
 
    /* Wait on and free execution pool */
    ff_vk_exec_pool_free(&ctx->s, &ctx->exec_pool);
 
    /* This also frees all references from this pool */
    av_frame_free(&ctx->common.layered_frame);
 
    av_buffer_pool_uninit(&ctx->buf_pool);
 
    ff_vk_video_common_uninit(s, &ctx->common);
 
    if (ctx->sd_ctx_free)
        ctx->sd_ctx_free(ctx);
 
    ff_vk_uninit(s);
}
 
static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_ref)
{
    int err;
    FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
    const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
    AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;
    AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
    AVVulkanDeviceContext *hwctx = device->hwctx;
    FFVulkanDecodeShared *ctx;
 
    if (dec->shared_ctx)
        return 0;
 
    dec->shared_ctx = av_refstruct_alloc_ext(sizeof(*ctx), 0, NULL,
                                             free_common);
    if (!dec->shared_ctx)
        return AVERROR(ENOMEM);
 
    ctx = dec->shared_ctx;
 
    ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
                                                 hwctx->nb_enabled_dev_extensions);
 
    if (vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
        if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
            av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
                   VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
            av_refstruct_unref(&dec->shared_ctx);
            return AVERROR(ENOSYS);
        }
    }
 
    err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);
    if (err < 0) {
        av_refstruct_unref(&dec->shared_ctx);
        return err;
    }
 
    return 0;
}
 
static VkResult vulkan_setup_profile(AVCodecContext *avctx,
                                     FFVulkanDecodeProfileData *prof,
                                     AVVulkanDeviceContext *hwctx,
                                     FFVulkanFunctions *vk,
                                     const FFVulkanDecodeDescriptor *vk_desc,
                                     VkVideoDecodeH264CapabilitiesKHR *h264_caps,
                                     VkVideoDecodeH265CapabilitiesKHR *h265_caps,
#if CONFIG_VP9_VULKAN_HWACCEL
                                     VkVideoDecodeVP9CapabilitiesKHR *vp9_caps,
#endif
                                     VkVideoDecodeAV1CapabilitiesKHR *av1_caps,
                                     VkVideoCapabilitiesKHR *caps,
                                     VkVideoDecodeCapabilitiesKHR *dec_caps,
                                     int cur_profile)
{
    VkVideoDecodeUsageInfoKHR *usage = &prof->usage;
    VkVideoProfileInfoKHR *profile = &prof->profile;
    VkVideoProfileListInfoKHR *profile_list = &prof->profile_list;
 
    VkVideoDecodeH264ProfileInfoKHR *h264_profile = &prof->h264_profile;
    VkVideoDecodeH265ProfileInfoKHR *h265_profile = &prof->h265_profile;
#if CONFIG_VP9_VULKAN_HWACCEL
    VkVideoDecodeVP9ProfileInfoKHR *vp9_profile  = &prof->vp9_profile;
#endif
    VkVideoDecodeAV1ProfileInfoKHR *av1_profile  = &prof->av1_profile;
 
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->sw_pix_fmt);
    if (!desc)
        return AVERROR(EINVAL);
 
    if (avctx->codec_id == AV_CODEC_ID_H264) {
        dec_caps->pNext = h264_caps;
        usage->pNext = h264_profile;
        h264_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR;
 
        /* Vulkan transmits all the constrant_set flags, rather than wanting them
         * merged in the profile IDC */
        h264_profile->stdProfileIdc = cur_profile & ~(AV_PROFILE_H264_CONSTRAINED |
                                                      AV_PROFILE_H264_INTRA);
 
        h264_profile->pictureLayout = avctx->field_order == AV_FIELD_UNKNOWN ||
                                      avctx->field_order == AV_FIELD_PROGRESSIVE ?
                                      VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_PROGRESSIVE_KHR :
                                      VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_INTERLEAVED_LINES_BIT_KHR;
    } else if (avctx->codec_id == AV_CODEC_ID_H265) {
        dec_caps->pNext = h265_caps;
        usage->pNext = h265_profile;
        h265_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR;
        h265_profile->stdProfileIdc = cur_profile;
#if CONFIG_VP9_VULKAN_HWACCEL
    } else if (avctx->codec_id == AV_CODEC_ID_VP9) {
        dec_caps->pNext = vp9_caps;
        usage->pNext = vp9_profile;
        vp9_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_VP9_PROFILE_INFO_KHR;
        vp9_profile->stdProfile = cur_profile;
#endif
    } else if (avctx->codec_id == AV_CODEC_ID_AV1) {
        dec_caps->pNext = av1_caps;
        usage->pNext = av1_profile;
        av1_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR;
        av1_profile->stdProfile = cur_profile;
        av1_profile->filmGrainSupport = !(avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN);
    }
 
    usage->sType           = VK_STRUCTURE_TYPE_VIDEO_DECODE_USAGE_INFO_KHR;
    usage->videoUsageHints = VK_VIDEO_DECODE_USAGE_DEFAULT_KHR;
 
    profile->sType               = VK_STRUCTURE_TYPE_VIDEO_PROFILE_INFO_KHR;
    profile->pNext               = usage;
    profile->videoCodecOperation = vk_desc->decode_op;
    profile->chromaSubsampling   = ff_vk_subsampling_from_av_desc(desc);
    profile->lumaBitDepth        = ff_vk_depth_from_av_depth(desc->comp[0].depth);
    profile->chromaBitDepth      = profile->lumaBitDepth;
 
    profile_list->sType        = VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR;
    profile_list->profileCount = 1;
    profile_list->pProfiles    = profile;
 
    /* Get the capabilities of the decoder for the given profile */
    caps->sType = VK_STRUCTURE_TYPE_VIDEO_CAPABILITIES_KHR;
    caps->pNext = dec_caps;
    dec_caps->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_CAPABILITIES_KHR;
    /* dec_caps->pNext already filled in */
 
    return vk->GetPhysicalDeviceVideoCapabilitiesKHR(hwctx->phys_dev, profile,
                                                     caps);
}
 
static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_ref,
                                     enum AVPixelFormat *pix_fmt, VkFormat *vk_fmt,
                                     FFVulkanDecodeProfileData *prof,
                                     int *dpb_dedicate)
{
    VkResult ret;
    int max_level, base_profile, cur_profile;
    const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
    AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;
    AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
    AVVulkanDeviceContext *hwctx = device->hwctx;
    enum AVPixelFormat source_format;
    enum AVPixelFormat best_format;
    VkFormat best_vkfmt;
 
    FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
    FFVulkanDecodeShared *ctx = dec->shared_ctx;
    FFVulkanFunctions *vk = &ctx->s.vkfn;
 
    VkVideoCapabilitiesKHR *caps = &ctx->caps;
    VkVideoDecodeCapabilitiesKHR *dec_caps = &ctx->dec_caps;
 
    VkVideoDecodeH264CapabilitiesKHR h264_caps = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_CAPABILITIES_KHR,
    };
    VkVideoDecodeH265CapabilitiesKHR h265_caps = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_CAPABILITIES_KHR,
    };
#if CONFIG_VP9_VULKAN_HWACCEL
    VkVideoDecodeVP9CapabilitiesKHR vp9_caps = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_VP9_CAPABILITIES_KHR,
    };
#endif
    VkVideoDecodeAV1CapabilitiesKHR av1_caps = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_CAPABILITIES_KHR,
    };
 
    VkPhysicalDeviceVideoFormatInfoKHR fmt_info = {
        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_FORMAT_INFO_KHR,
        .pNext = &prof->profile_list,
    };
    VkVideoFormatPropertiesKHR *ret_info;
    uint32_t nb_out_fmts = 0;
 
    if (!(vk_desc->decode_extension & ctx->s.extensions)) {
        av_log(avctx, AV_LOG_ERROR, "Device does not support decoding %s!\n",
               avcodec_get_name(avctx->codec_id));
        return AVERROR(ENOSYS);
    }
 
    cur_profile = avctx->profile;
    base_profile = avctx->codec_id == AV_CODEC_ID_H264 ? AV_PROFILE_H264_CONSTRAINED_BASELINE :
                   avctx->codec_id == AV_CODEC_ID_H265 ? AV_PROFILE_HEVC_MAIN :
#if CONFIG_VP9_VULKAN_HWACCEL
                   avctx->codec_id == AV_CODEC_ID_VP9  ? STD_VIDEO_VP9_PROFILE_0 :
#endif
                   avctx->codec_id == AV_CODEC_ID_AV1  ? STD_VIDEO_AV1_PROFILE_MAIN :
                   0;
 
    ret = vulkan_setup_profile(avctx, prof, hwctx, vk, vk_desc,
                               &h264_caps,
                               &h265_caps,
#if CONFIG_VP9_VULKAN_HWACCEL
                               &vp9_caps,
#endif
                               &av1_caps,
                               caps,
                               dec_caps,
                               cur_profile);
    if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR &&
        avctx->flags & AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH &&
        avctx->profile != base_profile) {
        av_log(avctx, AV_LOG_VERBOSE, "%s profile %s not supported, attempting "
               "again with profile %s\n",
               avcodec_get_name(avctx->codec_id),
               avcodec_profile_name(avctx->codec_id, cur_profile),
               avcodec_profile_name(avctx->codec_id, base_profile));
        cur_profile = base_profile;
        ret = vulkan_setup_profile(avctx, prof, hwctx, vk, vk_desc,
                                   &h264_caps,
                                   &h265_caps,
#if CONFIG_VP9_VULKAN_HWACCEL
                                   &vp9_caps,
#endif
                                   &av1_caps,
                                   caps,
                                   dec_caps,
                                   cur_profile);
    }
 
    if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR) {
        av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
               "%s profile \"%s\" not supported!\n",
               avcodec_get_name(avctx->codec_id),
               avcodec_profile_name(avctx->codec_id, cur_profile));
        return AVERROR(EINVAL);
    } else if (ret == VK_ERROR_VIDEO_PROFILE_FORMAT_NOT_SUPPORTED_KHR) {
        av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
               "format (%s) not supported!\n",
               av_get_pix_fmt_name(avctx->sw_pix_fmt));
        return AVERROR(EINVAL);
    } else if (ret == VK_ERROR_FEATURE_NOT_PRESENT ||
               ret == VK_ERROR_FORMAT_NOT_SUPPORTED) {
        return AVERROR(EINVAL);
    } else if (ret != VK_SUCCESS) {
        return AVERROR_EXTERNAL;
    }
 
    max_level = avctx->codec_id == AV_CODEC_ID_H264 ? ff_vk_h264_level_to_av(h264_caps.maxLevelIdc) :
                avctx->codec_id == AV_CODEC_ID_H265 ? ff_vk_h265_level_to_av(h265_caps.maxLevelIdc) :
#if CONFIG_VP9_VULKAN_HWACCEL
                avctx->codec_id == AV_CODEC_ID_VP9  ? vp9_caps.maxLevel :
#endif
                avctx->codec_id == AV_CODEC_ID_AV1  ? av1_caps.maxLevel :
                0;
 
    av_log(avctx, AV_LOG_VERBOSE, "Decoder capabilities for %s profile \"%s\":\n",
           avcodec_get_name(avctx->codec_id),
           avcodec_profile_name(avctx->codec_id, cur_profile));
    av_log(avctx, AV_LOG_VERBOSE, "    Maximum level: %i (stream %i)\n",
           max_level, avctx->level);
    av_log(avctx, AV_LOG_VERBOSE, "    Width: from %i to %i\n",
           caps->minCodedExtent.width, caps->maxCodedExtent.width);
    av_log(avctx, AV_LOG_VERBOSE, "    Height: from %i to %i\n",
           caps->minCodedExtent.height, caps->maxCodedExtent.height);
    av_log(avctx, AV_LOG_VERBOSE, "    Width alignment: %i\n",
           caps->pictureAccessGranularity.width);
    av_log(avctx, AV_LOG_VERBOSE, "    Height alignment: %i\n",
           caps->pictureAccessGranularity.height);
    av_log(avctx, AV_LOG_VERBOSE, "    Bitstream offset alignment: %"PRIu64"\n",
           caps->minBitstreamBufferOffsetAlignment);
    av_log(avctx, AV_LOG_VERBOSE, "    Bitstream size alignment: %"PRIu64"\n",
           caps->minBitstreamBufferSizeAlignment);
    av_log(avctx, AV_LOG_VERBOSE, "    Maximum references: %u\n",
           caps->maxDpbSlots);
    av_log(avctx, AV_LOG_VERBOSE, "    Maximum active references: %u\n",
           caps->maxActiveReferencePictures);
    av_log(avctx, AV_LOG_VERBOSE, "    Codec header name: '%s' (driver), '%s' (compiled)\n",
           caps->stdHeaderVersion.extensionName,
           vk_desc->ext_props.extensionName);
    av_log(avctx, AV_LOG_VERBOSE, "    Codec header version: %i.%i.%i (driver), %i.%i.%i (compiled)\n",
           CODEC_VER(caps->stdHeaderVersion.specVersion),
           CODEC_VER(vk_desc->ext_props.specVersion));
    av_log(avctx, AV_LOG_VERBOSE, "    Decode modes:%s%s%s\n",
           dec_caps->flags ? "" :
               " invalid",
           dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR ?
               " reuse_dst_dpb" : "",
           dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR ?
               " dedicated_dpb" : "");
    av_log(avctx, AV_LOG_VERBOSE, "    Capability flags:%s%s%s\n",
           caps->flags ? "" :
               " none",
           caps->flags & VK_VIDEO_CAPABILITY_PROTECTED_CONTENT_BIT_KHR ?
               " protected" : "",
           caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR ?
               " separate_references" : "");
 
    /* Check if decoding is possible with the given parameters */
    if (avctx->coded_width  < caps->minCodedExtent.width   ||
        avctx->coded_height < caps->minCodedExtent.height  ||
        avctx->coded_width  > caps->maxCodedExtent.width   ||
        avctx->coded_height > caps->maxCodedExtent.height)
        return AVERROR(EINVAL);
 
    if (!(avctx->hwaccel_flags & AV_HWACCEL_FLAG_IGNORE_LEVEL) &&
        avctx->level > max_level)
        return AVERROR(EINVAL);
 
    /* Some basic sanity checking */
    if (!(dec_caps->flags & (VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR |
                             VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR))) {
        av_log(avctx, AV_LOG_ERROR, "Buggy driver signals invalid decoding mode: neither "
               "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR nor "
               "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR are set!\n");
        return AVERROR_EXTERNAL;
    } else if ((dec_caps->flags & (VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR |
                                   VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR) ==
                                   VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR) &&
               !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR)) {
        av_log(avctx, AV_LOG_ERROR, "Cannot initialize Vulkan decoding session, buggy driver: "
               "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR set "
               "but VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR is unset!\n");
        return AVERROR_EXTERNAL;
    }
 
    dec->dedicated_dpb = !(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR);
    ctx->common.layered_dpb = !dec->dedicated_dpb ? 0 :
                              !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR);
 
    if (dec->dedicated_dpb) {
        fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
    } else {
        fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR |
                              VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
                              VK_IMAGE_USAGE_TRANSFER_SRC_BIT         |
                              VK_IMAGE_USAGE_SAMPLED_BIT;
 
        if ((ctx->s.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
            (ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1))
            fmt_info.imageUsage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
    }
 
    /* Get the format of the images necessary */
    ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(hwctx->phys_dev,
                                                        &fmt_info,
                                                        &nb_out_fmts, NULL);
    if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
        (!nb_out_fmts && ret == VK_SUCCESS)) {
        return AVERROR(EINVAL);
    } else if (ret != VK_SUCCESS) {
        av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
               ff_vk_ret2str(ret));
        return AVERROR_EXTERNAL;
    }
 
    ret_info = av_mallocz(sizeof(*ret_info)*nb_out_fmts);
    if (!ret_info)
        return AVERROR(ENOMEM);
 
    for (int i = 0; i < nb_out_fmts; i++)
        ret_info[i].sType = VK_STRUCTURE_TYPE_VIDEO_FORMAT_PROPERTIES_KHR;
 
    ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(hwctx->phys_dev,
                                                        &fmt_info,
                                                        &nb_out_fmts, ret_info);
    if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
        (!nb_out_fmts && ret == VK_SUCCESS)) {
        av_free(ret_info);
        return AVERROR(EINVAL);
    } else if (ret != VK_SUCCESS) {
        av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
               ff_vk_ret2str(ret));
        av_free(ret_info);
        return AVERROR_EXTERNAL;
    }
 
    /* Find a format to use */
    *pix_fmt = best_format = AV_PIX_FMT_NONE;
    *vk_fmt  = best_vkfmt = VK_FORMAT_UNDEFINED;
    source_format = avctx->sw_pix_fmt;
 
    av_log(avctx, AV_LOG_DEBUG, "Choosing best pixel format for decoding from %i:\n", nb_out_fmts);
    for (int i = 0; i < nb_out_fmts; i++) {
        enum AVPixelFormat tmp = ff_vk_pix_fmt_from_vkfmt(ret_info[i].format);
        if (tmp == AV_PIX_FMT_NONE) {
            av_log(avctx, AV_LOG_WARNING, "Invalid/unknown Vulkan format %i!\n", ret_info[i].format);
            continue;
        }
 
        best_format = av_find_best_pix_fmt_of_2(tmp, best_format, source_format, 0, NULL);
        if (tmp == best_format)
            best_vkfmt = ret_info[i].format;
 
        av_log(avctx, AV_LOG_DEBUG, "    %s%s (Vulkan ID: %i)\n",
               av_get_pix_fmt_name(tmp), tmp == best_format ? "*" : "",
               ret_info[i].format);
    }
 
    av_free(ret_info);
 
    if (best_format == AV_PIX_FMT_NONE) {
        av_log(avctx, AV_LOG_ERROR, "No valid/compatible pixel format found for decoding!\n");
        return AVERROR(EINVAL);
    } else {
        av_log(avctx, AV_LOG_VERBOSE, "Chosen frame pixfmt: %s (Vulkan ID: %i)\n",
               av_get_pix_fmt_name(best_format), best_vkfmt);
    }
 
    *pix_fmt = best_format;
    *vk_fmt = best_vkfmt;
 
    *dpb_dedicate = dec->dedicated_dpb;
 
    return 0;
}
 
static void free_profile_data(AVHWFramesContext *hwfc)
{
    av_free(hwfc->user_opaque);
}
 
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
{
    int err, dedicated_dpb;
    AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
    AVVulkanFramesContext *hwfc = frames_ctx->hwctx;
    FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
    FFVulkanDecodeProfileData *prof = NULL;
 
    err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);
    if (err < 0)
        return err;
 
    frames_ctx->sw_format = avctx->sw_pix_fmt;
 
    if (!DECODER_IS_SDR(avctx->codec_id)) {
        prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
        if (!prof)
            return AVERROR(ENOMEM);
 
        err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
                                        &frames_ctx->sw_format,
                                        &hwfc->format[0],
                                        prof, &dedicated_dpb);
        if (err < 0) {
            av_free(prof);
            return err;
        }
 
        frames_ctx->user_opaque = prof;
        frames_ctx->free        = free_profile_data;
 
        hwfc->create_pnext = &prof->profile_list;
    } else {
        hwfc->format[0] = VK_FORMAT_UNDEFINED;
        switch (frames_ctx->sw_format) {
        case AV_PIX_FMT_GBRAP16:
            /* This should be more efficient for downloading and using */
            frames_ctx->sw_format = AV_PIX_FMT_RGBA64;
            break;
        case AV_PIX_FMT_GBRP10:
            /* This saves memory bandwidth when downloading */
            frames_ctx->sw_format = AV_PIX_FMT_X2BGR10;
            break;
        case AV_PIX_FMT_RGB24:
        case AV_PIX_FMT_BGR0:
            /* mpv has issues with bgr0 mapping, so just remap it */
            frames_ctx->sw_format = AV_PIX_FMT_RGB0;
            break;
        /* DPX endian mismatch remappings */
        case AV_PIX_FMT_RGB48LE:
        case AV_PIX_FMT_RGB48BE: frames_ctx->sw_format = AV_PIX_FMT_GBRP16; break;
        case AV_PIX_FMT_RGBA64BE: frames_ctx->sw_format = AV_PIX_FMT_RGBA64; break;
        case AV_PIX_FMT_GRAY16BE: frames_ctx->sw_format = AV_PIX_FMT_GRAY16; break;
        /* ProRes needs to clear the input image, which is not possible on YUV formats */
        case AV_PIX_FMT_YUVA422P10:
        case AV_PIX_FMT_YUVA444P10:
        case AV_PIX_FMT_YUVA422P12:
        case AV_PIX_FMT_YUVA444P12:
            hwfc->format[3] = VK_FORMAT_R16_UNORM;
            /* fallthrough */
        case AV_PIX_FMT_YUV422P10:
        case AV_PIX_FMT_YUV444P10:
        case AV_PIX_FMT_YUV422P12:
        case AV_PIX_FMT_YUV444P12:
            hwfc->format[0] = VK_FORMAT_R16_UNORM;
            hwfc->format[1] = VK_FORMAT_R16_UNORM;
            hwfc->format[2] = VK_FORMAT_R16_UNORM;
            break;
        default:
            break;
        }
    }
 
    const AVPixFmtDescriptor *pdesc = av_pix_fmt_desc_get(frames_ctx->sw_format);
    frames_ctx->width  = FFALIGN(avctx->coded_width, 1 << pdesc->log2_chroma_w);
    frames_ctx->height = FFALIGN(avctx->coded_height, 1 << pdesc->log2_chroma_h);
    frames_ctx->format = AV_PIX_FMT_VULKAN;
 
    hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
    hwfc->usage  = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
                   VK_IMAGE_USAGE_STORAGE_BIT      |
                   VK_IMAGE_USAGE_SAMPLED_BIT;
 
    if (prof) {
        FFVulkanDecodeShared *ctx;
 
        hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
        if (!dec->dedicated_dpb)
            hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
 
        ctx = dec->shared_ctx;
        if ((ctx->s.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
            (ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1))
            hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
    }
 
    return err;
}
 
static void vk_decode_free_params(void *opaque, uint8_t *data)
{
    FFVulkanDecodeShared *ctx = opaque;
    FFVulkanFunctions *vk = &ctx->s.vkfn;
    VkVideoSessionParametersKHR *par = (VkVideoSessionParametersKHR *)data;
    vk->DestroyVideoSessionParametersKHR(ctx->s.hwctx->act_dev, *par,
                                         ctx->s.hwctx->alloc);
    av_free(par);
}
 
int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDecodeShared *ctx,
                               const VkVideoSessionParametersCreateInfoKHR *session_params_create)
{
    VkVideoSessionParametersKHR *par = av_malloc(sizeof(*par));
    const FFVulkanFunctions *vk = &ctx->s.vkfn;
    VkResult ret;
 
    if (!par)
        return AVERROR(ENOMEM);
 
    /* Create session parameters */
    ret = vk->CreateVideoSessionParametersKHR(ctx->s.hwctx->act_dev, session_params_create,
                                              ctx->s.hwctx->alloc, par);
    if (ret != VK_SUCCESS) {
        av_log(logctx, AV_LOG_ERROR, "Unable to create Vulkan video session parameters: %s!\n",
               ff_vk_ret2str(ret));
        av_free(par);
        return AVERROR_EXTERNAL;
    }
    *par_ref = av_buffer_create((uint8_t *)par, sizeof(*par),
                                vk_decode_free_params, ctx, 0);
    if (!*par_ref) {
        vk_decode_free_params(ctx, (uint8_t *)par);
        return AVERROR(ENOMEM);
    }
 
    return 0;
}
 
int ff_vk_decode_uninit(AVCodecContext *avctx)
{
    FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
 
    av_freep(&dec->hevc_headers);
    av_buffer_unref(&dec->session_params);
    av_refstruct_unref(&dec->shared_ctx);
    av_freep(&dec->slice_off);
    return 0;
}
 
int ff_vk_decode_init(AVCodecContext *avctx)
{
    int err;
    FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
    FFVulkanDecodeShared *ctx;
    FFVulkanContext *s;
    int async_depth;
    const VkVideoProfileInfoKHR *profile;
    const FFVulkanDecodeDescriptor *vk_desc;
 
    VkVideoSessionCreateInfoKHR session_create = {
        .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_CREATE_INFO_KHR,
    };
 
    err = ff_decode_get_hw_frames_ctx(avctx, AV_HWDEVICE_TYPE_VULKAN);
    if (err < 0)
        return err;
 
    /* Initialize contexts */
    ctx = dec->shared_ctx;
    s = &ctx->s;
 
    err = ff_vk_init(s, avctx, NULL, avctx->hw_frames_ctx);
    if (err < 0)
        return err;
 
    vk_desc = get_codecdesc(avctx->codec_id);
 
    profile = get_video_profile(ctx, avctx->codec_id);
    if ((vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) && !profile) {
        av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");
        return AVERROR(EINVAL);
    }
 
    /* Create queue context */
    vk_desc = get_codecdesc(avctx->codec_id);
    ctx->qf = ff_vk_qf_find(s, vk_desc->queue_flags, vk_desc->decode_op);
    if (!ctx->qf) {
        av_log(avctx, AV_LOG_ERROR, "Decoding of %s is not supported by this device\n",
               avcodec_get_name(avctx->codec_id));
        return err;
    }
 
    session_create.queueFamilyIndex = ctx->qf->idx;
    session_create.maxCodedExtent = ctx->caps.maxCodedExtent;
    session_create.maxDpbSlots = ctx->caps.maxDpbSlots;
    session_create.maxActiveReferencePictures = ctx->caps.maxActiveReferencePictures;
    session_create.pictureFormat = s->hwfc->format[0];
    session_create.referencePictureFormat = session_create.pictureFormat;
    session_create.pStdHeaderVersion = &vk_desc->ext_props;
    session_create.pVideoProfile = profile;
#ifdef VK_KHR_video_maintenance2
    if (ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_2)
        session_create.flags = VK_VIDEO_SESSION_CREATE_INLINE_SESSION_PARAMETERS_BIT_KHR;
#endif
 
    /* Create decode exec context for this specific main thread.
     * 2 async contexts per thread was experimentally determined to be optimal
     * for a majority of streams. */
    async_depth = 2*ctx->qf->num;
    /* We don't need more than 2 per thread context */
    async_depth = FFMIN(async_depth, 2*avctx->thread_count);
    /* Make sure there are enough async contexts for each thread */
    async_depth = FFMAX(async_depth, avctx->thread_count);
 
    err = ff_vk_exec_pool_init(s, ctx->qf, &ctx->exec_pool,
                               async_depth, 0, 0, 0, profile);
    if (err < 0)
        goto fail;
 
    if (!DECODER_IS_SDR(avctx->codec_id)) {
        err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
        if (err < 0)
            goto fail;
    }
 
    /* If doing an out-of-place decoding, create a DPB pool */
    if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) {
        AVHWFramesContext *dpb_frames;
        AVVulkanFramesContext *dpb_hwfc;
 
        ctx->common.dpb_hwfc_ref = av_hwframe_ctx_alloc(s->frames->device_ref);
        if (!ctx->common.dpb_hwfc_ref) {
            err = AVERROR(ENOMEM);
            goto fail;
        }
 
        dpb_frames = (AVHWFramesContext *)ctx->common.dpb_hwfc_ref->data;
        dpb_frames->format    = s->frames->format;
        dpb_frames->sw_format = s->frames->sw_format;
        dpb_frames->width     = s->frames->width;
        dpb_frames->height    = s->frames->height;
 
        dpb_hwfc = dpb_frames->hwctx;
        dpb_hwfc->create_pnext = (void *)ff_vk_find_struct(ctx->s.hwfc->create_pnext,
                                                           VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
        dpb_hwfc->format[0]    = s->hwfc->format[0];
        dpb_hwfc->tiling       = VK_IMAGE_TILING_OPTIMAL;
        dpb_hwfc->usage        = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
 
        if (ctx->common.layered_dpb)
            dpb_hwfc->nb_layers = ctx->caps.maxDpbSlots;
 
        err = av_hwframe_ctx_init(ctx->common.dpb_hwfc_ref);
        if (err < 0)
            goto fail;
 
        if (ctx->common.layered_dpb) {
            ctx->common.layered_frame = vk_get_dpb_pool(ctx);
            if (!ctx->common.layered_frame) {
                err = AVERROR(ENOMEM);
                goto fail;
            }
 
            err = ff_vk_create_view(&ctx->s, &ctx->common,
                                    &ctx->common.layered_view,
                                    &ctx->common.layered_aspect,
                                    (AVVkFrame *)ctx->common.layered_frame->data[0],
                                    s->hwfc->format[0], VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR);
            if (err < 0)
                goto fail;
        }
    }
 
    if (!DECODER_IS_SDR(avctx->codec_id)) {
        err = decode_reset(avctx, ctx);
        if (err < 0)
            return err;
    } else {
        /* For SDR decoders, this alignment value will be 0. Since this will make
         * add_slice() malfunction, set it to a sane default value. */
        ctx->caps.minBitstreamBufferSizeAlignment = AV_INPUT_BUFFER_PADDING_SIZE;
    }
 
    const VkPhysicalDeviceDriverProperties *driver_props;
    driver_props = &dec->shared_ctx->s.driver_props;
    if (driver_props->driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY &&
        driver_props->conformanceVersion.major == 1 &&
        driver_props->conformanceVersion.minor == 3 &&
        driver_props->conformanceVersion.subminor == 8 &&
        driver_props->conformanceVersion.patch < 3)
        dec->quirk_av1_offset = 1;
 
    av_log(avctx, AV_LOG_VERBOSE, "Vulkan decoder initialization successful\n");
 
    return 0;
 
fail:
    ff_vk_decode_uninit(avctx);
 
    return err;
}