如何读取以O_DIRECT打开且大小非块倍数的文件尾部？

问题描述 #

我用O_DIRECT标志打开文件并通过AIO读取，但当文件大小不是块大小的倍数时，不知道如何读取最后一部分。以下是运行失败的示例代码：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <linux/aio_abi.h>
#include <sys/syscall.h>
#include <errno.h>
#include <memory>
#include <utility>
#include <iostream>

template<typename FuncT> class ScopeExit {
public:
    explicit ScopeExit(FuncT&& fn) : f(std::forward<FuncT>(fn)) {}

    ~ScopeExit() { f(); }

    ScopeExit(const ScopeExit&) = delete;
    ScopeExit& operator=(const ScopeExit&) = delete;
    ScopeExit(ScopeExit&&) = delete;
    ScopeExit& operator=(ScopeExit&&) = delete;

private:
    FuncT f;
};
template<typename F> ScopeExit(F&&) -> ScopeExit<F>;

template<typename... Args> long checked_syscall(long syscall_number, Args... args) {
    long ret = syscall(syscall_number, args...);
    if (ret < 0) {
        auto error = errno;
        std::cerr << "Syscall " << syscall_number << " failed: " 
                  << strerror(error) << " (errno=" << error << ")\n";
        std::exit(1);
    }
    return ret;
}
    
int io_setup(unsigned nr_events, aio_context_t *ctx_idp) {
    return checked_syscall(__NR_io_setup, nr_events, ctx_idp);
}

int io_submit(aio_context_t ctx_id, long nr, struct iocb **iocbpp) {
    return checked_syscall(__NR_io_submit, ctx_id, nr, iocbpp);
}

int io_getevents(aio_context_t ctx_id, long min_nr, long nr,
                             struct io_event *events, struct timespec *timeout) {
    return checked_syscall(__NR_io_getevents, ctx_id, min_nr, nr, events, timeout);
}

int io_destroy(aio_context_t ctx_id) {
    return checked_syscall(__NR_io_destroy, ctx_id);
}

long checked_libcall(const char* msg, long ret) {
    if (ret < 0) {
        auto error = errno;
        std::cerr << msg << " failed: "  << strerror(error) << " (errno=" << error << ")\n";
        std::exit(1);
    }
    return ret; 
}

long checked_libcall(const char* msg, long want, long ret) {
    ret = checked_libcall(msg, ret);
    if (ret != want) {
        std::cerr << msg << " failed: want '" << want << "' got '" << ret << "'\n";
        std::exit(1);
    }
    return ret;
}
    
#define CHECKED_LIBCALL(...) checked_libcall(#__VA_ARGS__, (__VA_ARGS__))
    
int main() {
    const char *filename = "onebyte.txt";
    
    // First, create a file with one byte
    {
        int fd_write = CHECKED_LIBCALL(open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644));
        auto closeWrite = ScopeExit([&]{ close(fd_write); });
        char write_byte = 'A';
        CHECKED_LIBCALL(write(fd_write, &write_byte, 1), 1);
        CHECKED_LIBCALL(fsync(fd_write));
    }
    std::cout << "Created file with one byte\n";

    // Now read it using AIO
    // Initialize the AIO context
    aio_context_t ctx = 0;
    io_setup(128, &ctx);
    auto destroyCtx = ScopeExit([&]{ io_destroy(ctx); });
    
    // Open the file for reading
    int fd_read = CHECKED_LIBCALL(open(filename, O_RDONLY | O_DIRECT));
    auto closeRead = ScopeExit([&]{ close(fd_read); });

    // Prepare 4K aligned buffer for reading
    auto buffer = std::unique_ptr<char[]>(new (std::align_val_t{4 << 10}) char[1]);

    // Prepare the I/O control block
    struct iocb cb;
    struct iocb *cbs[1];
    memset(&cb, 0, sizeof(cb));
    cb.aio_fildes = fd_read;
    cb.aio_lio_opcode = IOCB_CMD_PREAD;
    cb.aio_buf = reinterpret_cast<uint64_t>(buffer.get());
    cb.aio_offset = 0;
    cb.aio_nbytes = 1;
    cbs[0] = &cb;

    // Submit the I/O request
    CHECKED_LIBCALL(io_submit(ctx, 1, cbs), 1);

    // Wait for completion
    struct io_event event;
    CHECKED_LIBCALL(io_getevents(ctx, 1, 1, &event, NULL), 1);

    // Check results
    if (event.res < 0) {
        auto error = -event.res;
        std::cerr << "AIO read error: errno=" << error << " [" << strerror(error) << "]\n";
    } else {
        std::cerr << "Read " << event.res << " byte: " << buffer[0] << "\n";
    }

    // remove the file
    unlink(filename);
    
    return 0;
}

失败原因 #

这段代码报错的核心是**O_DIRECT对IO操作有强制对齐要求**：

• 缓冲区内存地址必须对齐到文件系统的块大小
• 读写偏移量必须是块大小的整数倍
• 读写字节数必须是块大小的整数倍

示例中aio_nbytes设为1，不符合块大小（通常为4096字节）的整数倍要求，因此内核返回EINVAL错误。

解决方案 #

要处理非块大小倍数的文件，需按以下步骤调整：

1. 获取文件系统块大小：通过stat或fpathconf获取目标文件所在文件系统的最优IO块大小
2. 对齐IO请求参数：分配块大小整数倍的对齐缓冲区，按块大小设置读取长度
3. 提取有效数据：IO完成后，根据实际读取的字节数从缓冲区中提取需要的内容

修改后的可运行代码 #

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <linux/aio_abi.h>
#include <sys/syscall.h>
#include <errno.h>
#include <memory>
#include <utility>
#include <iostream>
#include <sys/stat.h>

template<typename FuncT> class ScopeExit {
public:
    explicit ScopeExit(FuncT&& fn) : f(std::forward<FuncT>(fn)) {}

    ~ScopeExit() { f(); }

    ScopeExit(const ScopeExit&) = delete;
    ScopeExit& operator=(const ScopeExit&) = delete;
    ScopeExit(ScopeExit&&) = delete;
    ScopeExit& operator=(ScopeExit&&) = delete;

private:
    FuncT f;
};
template<typename F> ScopeExit(F&&) -> ScopeExit<F>;

template<typename... Args> long checked_syscall(long syscall_number, Args... args) {
    long ret = syscall(syscall_number, args...);
    if (ret < 0) {
        auto error = errno;
        std::cerr << "Syscall " << syscall_number << " failed: " 
                  << strerror(error) << " (errno=" << error << ")\n";
        std::exit(1);
    }
    return ret;
}
    
int io_setup(unsigned nr_events, aio_context_t *ctx_idp) {
    return checked_syscall(__NR_io_setup, nr_events, ctx_idp);
}

int io_submit(aio_context_t ctx_id, long nr, struct iocb **iocbpp) {
    return checked_syscall(__NR_io_submit, ctx_id, nr, iocbpp);
}

int io_getevents(aio_context_t ctx_id, long min_nr, long nr,
                             struct io_event *events, struct timespec *timeout) {
    return checked_syscall(__NR_io_getevents, ctx_id, min_nr, nr, events, timeout);
}

int io_destroy(aio_context_t ctx_id) {
    return checked_syscall(__NR_io_destroy, ctx_id);
}

long checked_libcall(const char* msg, long ret) {
    if (ret < 0) {
        auto error = errno;
        std::cerr << msg << " failed: "  << strerror(error) << " (errno=" << error << ")\n";
        std::exit(1);
    }
    return ret; 
}

long checked_libcall(const char* msg, long want, long ret) {
    ret = checked_libcall(msg, ret);
    if (ret != want) {
        std::cerr << msg << " failed: want '" << want << "' got '" << ret << "'\n";
        std::exit(1);
    }
    return ret;
}
    
#define CHECKED_LIBCALL(...) checked_libcall(#__VA_ARGS__, (__VA_ARGS__))
    
int main() {
    const char *filename = "onebyte.txt";
    
    // First, create a file with one byte
    {
        int fd_write = CHECKED_LIBCALL(open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644));
        auto closeWrite = ScopeExit([&]{ close(fd_write); });
        char write_byte = 'A';
        CHECKED_LIBCALL(write(fd_write, &write_byte, 1), 1);
        CHECKED_LIBCALL(fsync(fd_write));
    }
    std::cout << "Created file with one byte\n";

    // Now read it using AIO
    // Initialize the AIO context
    aio_context_t ctx = 0;
    io_setup(128, &ctx);
    auto destroyCtx = ScopeExit([&]{ io_destroy(ctx); });
    
    // Open the file for reading
    int fd_read = CHECKED_LIBCALL(open(filename, O_RDONLY | O_DIRECT));
    auto closeRead = ScopeExit([&]{ close(fd_read); });

    // Get filesystem block size and file size
    struct stat sb;
    CHECKED_LIBCALL(fstat(fd_read, &sb));
    size_t block_size = sb.st_blksize;
    off_t file_size = sb.st_size;
    std::cout << "Filesystem block size: " << block_size << " bytes\n";
    std::cout << "File size: " << file_size << " bytes\n";

    // Prepare block-aligned buffer (size = block_size)
    auto buffer = std::unique_ptr<char[]>(new (std::align_val_t{block_size}) char[block_size]);

    // Prepare the I/O control block: read aligned length (block_size)
    struct iocb cb;
    struct iocb *cbs[1];
    memset(&cb, 0, sizeof(cb));
    cb.aio_fildes = fd_read;
    cb.aio_lio_opcode = IOCB_CMD_PREAD;
    cb.aio_buf = reinterpret_cast<uint64_t>(buffer.get());
    cb.aio_offset = 0;
    cb.aio_nbytes = block_size; // 按块大小读取，满足O_DIRECT要求
    cbs[0] = &cb;

    // Submit the I/O request
    CHECKED_LIBCALL(io_submit(ctx, 1, cbs), 1);

    // Wait for completion
    struct io_event event;
    CHECKED_LIBCALL(io_getevents(ctx, 1, 1, &event, NULL), 1);

    // Check results
    if (event.res < 0) {
        auto error = -event.res;
        std::cerr << "AIO read error: errno=" << error << " [" << strerror(error) << "]\n";
    } else {
        // 只提取实际需要的字节数
        std::cerr << "Read " << event.res << " byte(s): ";
        for (off_t i = 0; i < event.res; ++i) {
            std::cerr << buffer[i];
        }
        std::cerr << "\n";
    }

    // remove the file
    unlink(filename);
    
    return 0;
}

关键修改点 #

• 添加stat调用获取文件系统块大小st_blksize和文件实际大小st_size
• 缓冲区大小改为块大小，确保地址和长度都满足对齐要求
• aio_nbytes设为块大小，符合O_DIRECT的强制要求
• IO完成后，根据event.res（实际读取的字节数）提取有效数据

内容的提问来源于stack exchange，提问作者Hesky Fisher