qioring_p.h

Go to the documentation of this file.

// Copyright (C) 2025 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
// Qt-Security score:significant reason:default
 
#ifndef IOPROCESSOR_P_H
#define IOPROCESSOR_P_H
 
//
//  W A R N I N G
//  -------------
//
// This file is not part of the Qt API. It exists purely as an
// implementation detail. This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
//
 
#include <QtCore/private/qtcoreglobal_p.h>
 
#include <QtCore/qstring.h>
#include <QtCore/qspan.h>
#include <QtCore/qhash.h>
#include <QtCore/qfiledevice.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/qdeadlinetimer.h>
 
#ifdef Q_OS_LINUX
#  include <QtCore/qsocketnotifier.h>
struct io_uring_sqe;
struct io_uring_cqe;
#elif defined(Q_OS_WIN)
#  include <QtCore/qwineventnotifier.h>
#  include <qt_windows.h>
#  include <ioringapi.h>
#endif
 
#include <algorithm>
#include <filesystem>
#include <QtCore/qxpfunctional.h>
#include <variant>
#include <optional>
#include <type_traits>
 
/*
    This file defines an interface for the backend of QRandomAccessFile.
    The backends themselves are implemented in platform-specific files, such as
    ioring_linux.cpp, ioring_win.cpp, etc.
    And has a lower-level interface than the public interface will have, but the
    separation hopefully makes it easier to implement the ioring backends, test
    them, and tweak them without the higher-level interface needing to see
    changes, and to make it possible to tweak the higher-level interface without
    needing to touch the (somewhat similar) ioring backends.
 
    Most of the interface is just an enum QIORing::Operation + the
    QIORingRequest template class, which is specialized for each operation so it
    carries just the relevant data for that operation. And a small mechanism to
    store the request in a generic manner so they can be used in the
    implementation files at the cost of some overhead.
 
    There will be absolutely zero binary compatibility guarantees for this
    interface.
*/
 
QT_BEGIN_NAMESPACE
 
Q_DECLARE_LOGGING_CATEGORY(lcQIORing);
 
namespace QtPrivate {
Q_NAMESPACE
 
#define FOREACH_IO_OPERATION(OP)
    OP(Open)
    OP(Close)
    OP(Read)
    OP(Write)
    OP(VectoredRead)
    OP(VectoredWrite)
    OP(Flush)
    OP(Stat)
    OP(Cancel)
    /**/
#define DEFINE_ENTRY(OP) OP,
 
// clang-format off
enum class Operation : quint8 {
    FOREACH_IO_OPERATION(DEFINE_ENTRY)
 
    NumOperations,
};
// clang-format on
Q_ENUM_NS(Operation);
#undef DEFINE_ENTRY
 
#ifdef Q_OS_WIN
struct IORingApiTable;
#endif
 
#ifdef Q_OS_LINUX
extern Q_CORE_EXPORT std::atomic<qsizetype> testMaxReadWriteLen;
#endif
}; // namespace QtPrivate
 
template <QtPrivate::Operation Op>
struct QIORingRequest;
 
class QIORing final
{
    class GenericRequestType;
    struct RequestHandleTag; // Just used as an opaque pointer
public:
    static constexpr quint32 DefaultSubmissionQueueSize = 128;
    static constexpr quint32 DefaultCompletionQueueSize = DefaultSubmissionQueueSize * 2;
    using Operation = QtPrivate::Operation;
    using RequestHandle = RequestHandleTag *;
 
    Q_CORE_EXPORT
    explicit QIORing(quint32 submissionQueueSize = DefaultSubmissionQueueSize,
                     quint32 completionQueueSize = DefaultCompletionQueueSize);
    Q_CORE_EXPORT
    ~QIORing();
    Q_DISABLE_COPY_MOVE(QIORing)
 
    Q_CORE_EXPORT
    static QIORing *sharedInstance();
    bool ensureInitialized() { return initializeIORing(); }
 
    Q_CORE_EXPORT
    static bool supportsOperation(Operation op);
    template <Operation Op>
    QIORing::RequestHandle queueRequest(QIORingRequest<Op> &&request)
    {
        Q_ASSERT(supportsOperation(Op));
        auto &r = pendingRequests.emplace_back(std::move(request));
        addrItMap.emplace(&r, std::prev(pendingRequests.end()));
        if (queueRequestInternal(r) == QueuedRequestStatus::CompletedImmediately)
            return nullptr; // Return an invalid handle, to avoid ABA with following requests
        return reinterpret_cast<RequestHandle>(&r);
    }
    Q_CORE_EXPORT
    void submitRequests();
    Q_CORE_EXPORT
    bool waitForRequest(RequestHandle handle, QDeadlineTimer deadline = QDeadlineTimer::Forever);
 
    quint32 submissionQueueSize() const noexcept { return sqEntries; }
    quint32 completionQueueSize() const noexcept { return cqEntries; }
 
private:
    std::list<GenericRequestType> pendingRequests;
    using PendingRequestsIterator = decltype(pendingRequests.begin());
    QHash<void *, PendingRequestsIterator> addrItMap;
    std::optional<PendingRequestsIterator> lastUnqueuedIterator;
    quint32 sqEntries = 0;
    quint32 cqEntries = 0;
    quint32 inFlightRequests = 0;
    quint32 unstagedRequests = 0;
    bool stagePending = false;
    bool preparingRequests = false;
    qsizetype ongoingSplitOperations = 0;
 
    Q_CORE_EXPORT
    bool initializeIORing();
 
    enum class QueuedRequestStatus : bool {
        Pending = false,
        CompletedImmediately = true,
    };
    Q_CORE_EXPORT
    QueuedRequestStatus queueRequestInternal(GenericRequestType &request);
    void prepareRequests();
    void completionReady();
    bool waitForCompletions(QDeadlineTimer deadline);
 
    template <typename Fun>
    static auto invokeOnOp(GenericRequestType &req, Fun fn);
 
    template <Operation Op>
    static void setFileErrorResult(QIORingRequest<Op> &req, QFileDevice::FileError error)
    {
        req.result.template emplace<QFileDevice::FileError>(error);
    }
    static void setFileErrorResult(GenericRequestType &req, QFileDevice::FileError error);
    static void finishRequestWithError(GenericRequestType &req, QFileDevice::FileError error);
    static bool verifyFd(GenericRequestType &req);
 
    enum RequestPrepResult : quint8 {
        Ok,
        QueueFull,
        Defer,
        RequestCompleted,
    };
    enum class ReadWriteStatus : bool {
        MoreToDo,
        Finished,
    };
#ifdef Q_OS_LINUX
    using NativeResultType = qint32;
    static constexpr bool isResultFailure(NativeResultType result) { return result < 0; }
 
    std::optional<QSocketNotifier> notifier;
    // io_uring 'sq', 'sqe', and 'cqe' pointers:
    void *submissionQueue = nullptr;
    io_uring_sqe *submissionQueueEntries = nullptr;
    const io_uring_cqe *completionQueueEntries = nullptr;
 
    // Some pointers for working with the ring-buffer.
    // The pointers to const are controlled by the kernel.
    const quint32 *sqHead = nullptr;
    quint32 *sqTail = nullptr;
    const quint32 *sqIndexMask = nullptr;
    quint32 *sqIndexArray = nullptr;
    quint32 *cqHead = nullptr;
    const quint32 *cqTail = nullptr;
    const quint32 *cqIndexMask = nullptr;
    // Because we want the flush to act as a barrier operation we need to track
    // if there is one currently in progress. With kernel 6.16+ this seems to be
    // fixed, but since we support older kernels we implement this deferring
    // ourselves.
    bool flushInProgress = false;
 
    int io_uringFd = -1;
    int eventDescriptor = -1;
    [[nodiscard]]
    RequestPrepResult prepareRequest(io_uring_sqe *sqe, GenericRequestType &request);
 
    template <typename SpanOfBytes>
    auto getVectoredOpAddressAndSize(QIORing::GenericRequestType &request,
                                     QSpan<SpanOfBytes> spans);
#elif defined(Q_OS_WIN)
    // We use UINT32 because that's the type used for size parameters in their API.
    static constexpr qsizetype MaxReadWriteLen = std::numeric_limits<UINT32>::max();
    using NativeResultType = HRESULT;
    static constexpr bool isResultFailure(NativeResultType result)
    {
        return FAILED(result) && result != HRESULT_FROM_WIN32(ERROR_HANDLE_EOF);
    }
 
    std::optional<QWinEventNotifier> notifier;
    HIORING ioRingHandle = nullptr;
    HANDLE eventHandle = INVALID_HANDLE_VALUE;
    const QtPrivate::IORingApiTable *apiTable;
 
    bool initialized = false;
    bool queueWasFull = false;
 
    [[nodiscard]]
    RequestPrepResult prepareRequest(GenericRequestType &request);
#endif
    static QFileDevice::FileError mapFileError(NativeResultType result,
                                               QFileDevice::FileError defaultValue);
    using SetResultFn = qxp::function_ref<qint64(qint64)>;
    static ReadWriteStatus handleReadCompletion(size_t value, QSpan<std::byte> *destinations,
                                                void *voidExtra, SetResultFn setResult);
    template <Operation Op>
    ReadWriteStatus handleReadCompletion(NativeResultType result, size_t value,
                                         GenericRequestType *request);
    static ReadWriteStatus handleWriteCompletion(size_t value,
                                                 const QSpan<const std::byte> *sources,
                                                 void *voidExtra, SetResultFn setResult);
    template <Operation Op>
    ReadWriteStatus handleWriteCompletion(NativeResultType result, size_t value,
                                          GenericRequestType *request);
    void finalizeReadWriteCompletion(GenericRequestType *request, ReadWriteStatus rwstatus);
};
 
struct QIORingRequestEmptyBase
{
};
 
template <QtPrivate::Operation Op>
struct QIORingResult;
template <QtPrivate::Operation Op>
struct QIORingRequest;
 
// @todo: q23::expected once emplace() returns a reference
template <QtPrivate::Operation Op>
using ExpectedResultType = std::variant<std::monostate, QIORingResult<Op>, QFileDevice::FileError>;
 
struct QIORingRequestOffsetFdBase : QIORingRequestEmptyBase
{
    qintptr fd;
    quint64 offset;
};
 
template <QtPrivate::Operation Op, typename Base = QIORingRequestOffsetFdBase>
struct QIORingRequestBase : Base
{
    ExpectedResultType<Op> result; // To be filled in by the backend
    QtPrivate::SlotObjUniquePtr callback;
    template <typename Func>
    Q_ALWAYS_INLINE void setCallback(Func &&func)
    {
        using Prototype = void (*)(const QIORingRequest<Op> &);
        callback.reset(QtPrivate::makeCallableObject<Prototype>(std::forward<Func>(func)));
    }
};
 
template <>
struct QIORingResult<QtPrivate::Operation::Open>
{
    // On Windows this is a HANDLE
    qintptr fd;
};
template <>
struct QIORingRequest<QtPrivate::Operation::Open> final
    : QIORingRequestBase<QtPrivate::Operation::Open, QIORingRequestEmptyBase>
{
    std::filesystem::path path;
    QFileDevice::OpenMode flags;
};
template <>
struct QIORingResult<QtPrivate::Operation::Close>
{
};
template <>
struct QIORingRequest<QtPrivate::Operation::Close> final
    : QIORingRequestBase<QtPrivate::Operation::Close, QIORingRequestEmptyBase>
{
    // On Windows this is a HANDLE
    qintptr fd;
};
 
template <>
struct QIORingResult<QtPrivate::Operation::Write>
{
    qint64 bytesWritten;
};
template <>
struct QIORingRequest<QtPrivate::Operation::Write> final
    : QIORingRequestBase<QtPrivate::Operation::Write>
{
    QSpan<const std::byte> source;
};
template <>
struct QIORingResult<QtPrivate::Operation::VectoredWrite> final
    : QIORingResult<QtPrivate::Operation::Write>
{
};
template <>
struct QIORingRequest<QtPrivate::Operation::VectoredWrite> final
    : QIORingRequestBase<QtPrivate::Operation::VectoredWrite>
{
    QSpan<const QSpan<const std::byte>> sources;
};
 
template <>
struct QIORingResult<QtPrivate::Operation::Read>
{
    qint64 bytesRead;
};
template <>
struct QIORingRequest<QtPrivate::Operation::Read> final
    : QIORingRequestBase<QtPrivate::Operation::Read>
{
    QSpan<std::byte> destination;
};
 
template <>
struct QIORingResult<QtPrivate::Operation::VectoredRead> final
    : QIORingResult<QtPrivate::Operation::Read>
{
};
template <>
struct QIORingRequest<QtPrivate::Operation::VectoredRead> final
    : QIORingRequestBase<QtPrivate::Operation::VectoredRead>
{
    QSpan<QSpan<std::byte>> destinations;
};
 
template <>
struct QIORingResult<QtPrivate::Operation::Flush> final
{
    // No value in the result, just a success or failure
};
template <>
struct QIORingRequest<QtPrivate::Operation::Flush> final : QIORingRequestBase<QtPrivate::Operation::Flush, QIORingRequestEmptyBase>
{
    // On Windows this is a HANDLE
    qintptr fd;
};
 
template <>
struct QIORingResult<QtPrivate::Operation::Stat> final
{
    quint64 size;
};
template <>
struct QIORingRequest<QtPrivate::Operation::Stat> final
    : QIORingRequestBase<QtPrivate::Operation::Stat, QIORingRequestEmptyBase>
{
    // On Windows this is a HANDLE
    qintptr fd;
};
 
// This is not inheriting the QIORingRequestBase because it doesn't have a result,
// whether it was successful or not is indicated by whether the operation
// it was cancelling was successful or not.
template <>
struct QIORingRequest<QtPrivate::Operation::Cancel> final : QIORingRequestEmptyBase
{
    QIORing::RequestHandle handle;
    QtPrivate::SlotObjUniquePtr callback;
    template <typename Func>
    Q_ALWAYS_INLINE void setCallback(Func &&func)
    {
        using Op = QtPrivate::Operation;
        using Prototype = void (*)(const QIORingRequest<Op::Cancel> &);
        callback.reset(QtPrivate::makeCallableObject<Prototype>(std::forward<Func>(func)));
    }
};
 
template <QIORing::Operation Op>
Q_ALWAYS_INLINE void invokeCallback(const QIORingRequest<Op> &request)
{
    if (!request.callback)
        return;
    void *args[2] = { nullptr, const_cast<QIORingRequest<Op> *>(&request) };
    request.callback->call(nullptr, args);
}
 
class QIORing::GenericRequestType
{
    friend class QIORing;
 
#define POPULATE_VARIANT(Op)
    QIORingRequest<Operation::Op>,
    /**/
 
    std::variant<
        FOREACH_IO_OPERATION(POPULATE_VARIANT)
        std::monostate
    > taggedUnion;
 
#undef POPULATE_VARIANT
 
    void *extraData = nullptr;
    bool queued = false;
 
    template <Operation Op>
    Q_ALWAYS_INLINE void initializeStorage(QIORingRequest<Op> &&t) noexcept
    {
        static_assert(Op < Operation::NumOperations);
        taggedUnion.emplace<QIORingRequest<Op>>(std::move(t));
    }
 
    Q_CORE_EXPORT
    static void cleanupExtra(Operation op, void *extra);
    template <typename T>
    T *getOrInitializeExtra()
    {
        if (!extraData)
            extraData = new T();
        return static_cast<T *>(extraData);
    }
    template <typename T>
    T *getExtra() const
    {
        return static_cast<T *>(extraData);
    }
    void reset() noexcept
    {
        Operation op = operation();
        taggedUnion.emplace<std::monostate>();
        if (extraData)
            cleanupExtra(op, std::exchange(extraData, nullptr));
    }
 
public:
    template <Operation Op>
    explicit GenericRequestType(QIORingRequest<Op> &&t) noexcept
    {
        initializeStorage(std::move(t));
    }
    ~GenericRequestType() noexcept
    {
        reset();
    }
    Q_DISABLE_COPY_MOVE(GenericRequestType)
    // We have to provide equality operators. Since copying is disabled, we just check for equality
    // based on the address in memory. Two requests could be constructed to be equal, but we don't
    // actually care because the order in which they are added to the queue may also matter.
    friend bool operator==(const GenericRequestType &l, const GenericRequestType &r) noexcept
    {
        return std::addressof(l) == std::addressof(r);
    }
    friend bool operator!=(const GenericRequestType &l, const GenericRequestType &r) noexcept
    {
        return !(l == r);
    }
 
    Operation operation() const { return Operation(taggedUnion.index()); }
    template <Operation Op>
    QIORingRequest<Op> *requestData()
    {
        if (operation() == Op)
            return std::get_if<QIORingRequest<Op>>(&taggedUnion);
        Q_ASSERT("Wrong operation requested, see operation()");
        return nullptr;
    }
    template <Operation Op>
    QIORingRequest<Op> takeRequestData()
    {
        if (operation() == Op)
            return std::move(*std::get_if<QIORingRequest<Op>>(&taggedUnion));
        Q_ASSERT("Wrong operation requested, see operation()");
        return {};
    }
    bool wasQueued() const { return queued; }
    void setQueued(bool status) { queued = status; }
};
 
template <typename Fun>
auto QIORing::invokeOnOp(GenericRequestType &req, Fun fn)
{
#define INVOKE_ON_OP(Op)                           case
     QIORing::Operation::Op:
    fn(req.template requestData<Operation::Op>());
    return;
    /**/
 
    switch (req.operation()) {
        FOREACH_IO_OPERATION(INVOKE_ON_OP)
    case QIORing::Operation::NumOperations:
        break;
    }
 
    Q_UNREACHABLE();
#undef INVOKE_ON_OP
}
 
template <QIORing::Operation Op>
QIORing::ReadWriteStatus QIORing::handleReadCompletion(NativeResultType result, size_t value,
                                                       GenericRequestType *request)
{
    static_assert(Op == Operation::Read || Op == Operation::VectoredRead);
    QIORingRequest<Op> *readRequest = request->requestData<Op>();
    Q_ASSERT(readRequest);
 
    if (isResultFailure(result)) { // error
        QFileDevice::FileError fileError = mapFileError(result, QFileDevice::ReadError);
        QIORing::setFileErrorResult(*readRequest, fileError);
        return ReadWriteStatus::Finished;
    }
 
    auto setResult = [readRequest](qint64 bytesRead) {
        auto &readResult = [&readRequest]() -> QIORingResult<Op> & {
            if (auto *result = std::get_if<QIORingResult<Op>>(&readRequest->result))
                return *result;
            return readRequest->result.template emplace<QIORingResult<Op>>();
        }();
        readResult.bytesRead += bytesRead;
        return readResult.bytesRead;
    };
 
    auto *destinations = [&readRequest]() {
        if constexpr (Op == Operation::Read)
            return &readRequest->destination;
        else
            return &readRequest->destinations[0];
    }();
 
    QIORing::ReadWriteStatus rwstatus = handleReadCompletion(value, destinations,
                                                             request->getExtra<void>(), setResult);
    finalizeReadWriteCompletion(request, rwstatus);
    return rwstatus;
}
 
template <QIORing::Operation Op>
QIORing::ReadWriteStatus QIORing::handleWriteCompletion(NativeResultType result, size_t value,
                                                        GenericRequestType *request)
{
    static_assert(Op == Operation::Write || Op == Operation::VectoredWrite);
    QIORingRequest<Op> *writeRequest = request->requestData<Op>();
    Q_ASSERT(writeRequest);
 
    if (isResultFailure(result)) { // error
        QFileDevice::FileError fileError = mapFileError(result, QFileDevice::WriteError);
        QIORing::setFileErrorResult(*writeRequest, fileError);
        return ReadWriteStatus::Finished;
    }
 
    auto setResult = [writeRequest](qint64 bytesWritten) {
        auto &writeResult = [&writeRequest]() -> QIORingResult<Op> & {
            if (auto *result = std::get_if<QIORingResult<Op>>(&writeRequest->result))
                return *result;
            return writeRequest->result.template emplace<QIORingResult<Op>>();
        }();
        writeResult.bytesWritten += bytesWritten;
        return writeResult.bytesWritten;
    };
    auto *sources = [&writeRequest]() {
        if constexpr (Op == Operation::Write)
            return &writeRequest->source;
        else
            return &writeRequest->sources[0];
    }();
    QIORing::ReadWriteStatus rwstatus = handleWriteCompletion(value, sources,
                                                              request->getExtra<void>(), setResult);
    finalizeReadWriteCompletion(request, rwstatus);
    return rwstatus;
}
 
namespace QtPrivate {
// The 'extra' struct for Read/Write operations that must be split up
struct ReadWriteExtra
{
    qint64 totalProcessed = 0;
    qsizetype spanIndex = 0;
    qsizetype spanOffset = 0;
    qsizetype numSpans = 1;
};
} // namespace QtPrivate
 
QT_END_NAMESPACE
 
#endif // IOPROCESSOR_P_H