C语言映射1TB文件遇ENOMEM错误，如何解决？

系统为64位，RLIMIT_AS值设为18446744073709551615，ulimit -v显示虚拟内存无限制，但使用C语言mmap映射1TB文件时返回ENOMEM（错误码12，无法分配内存）。该文件由Python的np.lib.format.open_memmap创建，Python中用numpy.load(mmap_mode='r')可正常读取。

复现代码 #

创建numpy数组的Python代码 #

import numpy as np

shape = (75000, 5000000)
filename = 'datafile.obj'

if __name__ == '__main__':
  arr = np.lib.format.open_memmap(filename, mode='w+', dtype=np.float32, shape=shape)

读取文件的C代码 #

#include <stdbool.h>
#include <assert.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdint.h>
#include <unistd.h>

#include <sys/time.h>
#include <sys/resource.h>

#include <stdio.h>
#include <errno.h>

typedef enum {
  CNPY_LE, /* little endian (least significant byte to most significant byte) */
  CNPY_BE, /* big endian (most significant byte to least significant byte) */
  CNPY_NE, /* no / neutral endianness (each element is a single byte) */
  /* Host endianness is not supported because it is an incredibly bad idea to
     use it for storage. */
} cnpy_byte_order;

typedef enum {
  CNPY_B = 0, /* We want to use the values as index to the following arrays. */
  CNPY_I1,
  CNPY_I2,
  CNPY_I4,
  CNPY_I8,
  CNPY_U1,
  CNPY_U2,
  CNPY_U4,
  CNPY_U8,
  CNPY_F4,
  CNPY_F8,
  CNPY_C8,
  CNPY_C16,
} cnpy_dtype;

typedef enum {
  CNPY_C_ORDER,       /* C order (row major) */
  CNPY_FORTRAN_ORDER, /* Fortran order (column major) */
} cnpy_flat_order;

typedef enum {
  CNPY_SUCCESS,      /* success */
  CNPY_ERROR_FILE,   /* some error regarding handling of a file */
  CNPY_ERROR_MMAP,   /* some error regarding mmaping a file */
  CNPY_ERROR_FORMAT, /* file format error while reading some file */
} cnpy_status;

#define CNPY_MAX_DIM 4
typedef struct {
  cnpy_byte_order byte_order;
  cnpy_dtype dtype;
  cnpy_flat_order order;
  size_t n_dim;
  size_t dims[CNPY_MAX_DIM];
  char *raw_data;
  size_t data_begin;
  size_t raw_data_size;
} cnpy_array;

cnpy_status cnpy_open(const char * const fn, bool writable, cnpy_array *arr) {
  assert(arr != NULL);

  cnpy_array tmp_arr;

  /* open, mmap, and close the file */
  int fd = open(fn, writable? O_RDWR : O_RDONLY);
  if (fd == -1) {
    return CNPY_ERROR_FILE;
  }
  size_t raw_data_size = (size_t) lseek(fd, 0, SEEK_END);
  lseek(fd, 0, SEEK_SET);
  printf("%lu\n", raw_data_size);
  if (raw_data_size == 0) {
    close(fd); /* no point in checking for errors */
    return CNPY_ERROR_FORMAT;
  }
  if (raw_data_size == SIZE_MAX) {
    /* This is just because the author is too lazy to check for overflow on every pos+1 calculation. */
    close(fd);
    return CNPY_ERROR_FORMAT;
  }

  void *raw_data = mmap(
    NULL,
    raw_data_size,
    PROT_READ | PROT_WRITE,
    writable? MAP_SHARED : MAP_PRIVATE,
    fd,
    0 
  );

  if (raw_data == MAP_FAILED) {
    close(fd);
    return CNPY_ERROR_MMAP;
  }

  if (close(fd) != 0) {
    munmap(raw_data, raw_data_size);
    return CNPY_ERROR_FILE;
  }

  /* parse the file */
  // cnpy_status status = cnpy_parse(raw_data, raw_data_size, &tmp_arr); // library call ignore
  // if (status != CNPY_SUCCESS) {
  //   munmap(raw_data, raw_data_size);
  //   return status;
  // }
  // *arr = tmp_arr;

  return CNPY_SUCCESS;
}

int main(){

  cnpy_array arr = {};
  cnpy_status status = cnpy_open("datafile.obj", false, &arr);

  printf("status %i\n",(int) status);
  if(status != CNPY_SUCCESS){
    printf("failure\n");
    printf("errno %i\n", errno);
  }


    struct rlimit lim;
  printf("getrlimit RLIMIT_AS %s\n", (getrlimit(RLIMIT_AS, &lim) == 0 ? "success" : "failure") );
  printf("lim.rlim_cur %lu\n", lim.rlim_cur );
  printf("lim.rlim_max %lu\n", lim.rlim_max );
  printf("RLIM_INFINITY; %lu\n", RLIM_INFINITY );


  return 0;
}

编译命令 #

gcc -std=c11 -o mmap_testing main.c

注：使用~quf/cnpy库处理numpy相关格式。

1. 匹配文件打开权限与映射权限
   代码中无论writable参数是否为false，都给了PROT_READ | PROT_WRITE权限，但当以O_RDONLY打开文件时，请求写权限会触发ENOMEM——因为私有映射的写时复制需要内核分配匿名页后备，而只读文件不允许此操作。修改mmap的权限参数：

   int prot = writable ? (PROT_READ | PROT_WRITE) : PROT_READ;
   void *raw_data = mmap(
     NULL,
     raw_data_size,
     prot,
     writable? MAP_SHARED : MAP_PRIVATE,
     fd,
     0 
   );
   

2. 用fstat替代lseek获取文件大小
   lseek处理超大文件时可能存在兼容性问题，改用fstat更可靠：

   struct stat sb;
   if (fstat(fd, &sb) == -1) {
       close(fd);
       return CNPY_ERROR_FILE;
   }
   size_t raw_data_size = sb.st_size;
   

3. 排查内核映射数量限制
   即使RLIMIT_AS无限制，内核的vm.max_map_count参数可能限制进程的映射数量。查看当前值：

   sysctl vm.max_map_count
   

   若值过小，可临时调整：

   sudo sysctl -w vm.max_map_count=262144
   

   注：1TB文件仅需一个映射，此问题概率较低，但仍需排查。

内容的提问来源于stack exchange，提问作者Trick