R语言textreuse::TextReuseCorpus传变量参数时报非数值错误

现象 #

使用textreuse库实现minhash算法时，直接给TextReuseCorpus传入字面量参数n = 2L可以得到正常有效结果，可运行代码如下：

library(janeaustenr)
library(dplyr)
library(textreuse)
library(tibble)
library(tokenizers)
library(assertthat)
custom_tokenize_ngrams <- function(string, lowercase = TRUE, n = 3) {
  assertthat::assert_that(length(string) == 1)
  tokenizers::tokenize_ngrams(x = string, lowercase = lowercase, n = n) |> unlist()
}

janeaustenr::austen_books() |>
  dplyr::mutate(i_row = dplyr::row_number()) |> 
  dplyr::select(i_row, text) |>
  dplyr::mutate_all(as.character) |>
  dplyr::filter(nchar(text)>50 & nchar(text)<1000) |>
  utils::head(500) |>
  tibble::deframe() |>
  {\(.) 
    textreuse::TextReuseCorpus(
      text = ., 
      tokenizer = custom_tokenize_ngrams, 
      minhash_func = textreuse::minhash_generator(n = 60, seed = 123456) , 
      keep_tokens = TRUE,
      progress = TRUE, 
      n = 2L
    ) 
  }()

如果将n的取值存入变量，再传入TextReuseCorpus，逻辑完全一致的代码会触发报错，问题代码如下：

library(janeaustenr)
library(dplyr)
library(textreuse)
library(tibble)
library(tokenizers)
library(assertthat)
custom_tokenize_ngrams <- function(string, lowercase = TRUE, n = 3) {
  assertthat::assert_that(length(string) == 1)
  tokenizers::tokenize_ngrams(x = string, lowercase = lowercase, n = n) |> unlist()
}


nx = 2L

janeaustenr::austen_books() |>
  dplyr::mutate(i_row = dplyr::row_number()) |> 
  dplyr::select(i_row, text) |>
  dplyr::mutate_all(as.character) |>
  dplyr::filter(nchar(text)>50 & nchar(text)<1000) |>
  utils::head(500) |>
  tibble::deframe() |>
  {\(.) 
    textreuse::TextReuseCorpus(
      text = ., 
      tokenizer = custom_tokenize_ngrams, 
      minhash_func = textreuse::minhash_generator(n = 60, seed = 123456) , 
      keep_tokens = TRUE,
      progress = TRUE, 
      n = nx
    ) 
  }()

触发的报错信息为：

Error in n_call + 1 : non-numeric argument to binary operator

报错原因 #

TextReuseCorpus内部使用非标准计算处理传给分词器的额外参数，不会在调用环境中对传入的参数表达式提前求值。当传入n = nx时，函数内部拿到的是变量名nx这个符号对象，而不是变量存储的数值2L，后续执行n_call + 1的运算时，因为操作数不是数值类型就会抛出错误。

修复方案 #

• 方案1：使用函数工厂生成分词器，提前把n值固定在分词器的环境中，不通过TextReuseCorpus的...传n参数：

# 生成分词器的工厂函数
make_ngram_tokenizer <- function(n = 3, lowercase = TRUE) {
  function(string) {
    assertthat::assert_that(length(string) == 1)
    tokenizers::tokenize_ngrams(x = string, lowercase = lowercase, n = n) |> unlist()
  }
}

nx <- 2L
my_tokenizer <- make_ngram_tokenizer(n = nx)

# 调用时直接传入生成好的分词器，不需要再传n参数
janeaustenr::austen_books() |>
  dplyr::mutate(i_row = dplyr::row_number()) |> 
  dplyr::select(i_row, text) |>
  dplyr::mutate_all(as.character) |>
  dplyr::filter(nchar(text)>50 & nchar(text)<1000) |>
  utils::head(500) |>
  tibble::deframe() |>
  {\(.) 
    textreuse::TextReuseCorpus(
      text = ., 
      tokenizer = my_tokenizer, 
      minhash_func = textreuse::minhash_generator(n = 60, seed = 123456) , 
      keep_tokens = TRUE,
      progress = TRUE
    ) 
  }()

• 方案2：用do.call调用TextReuseCorpus，提前对所有参数求值，绕过非标准计算的问题：

nx <- 2L
input_text <- janeaustenr::austen_books() |>
  dplyr::mutate(i_row = dplyr::row_number()) |> 
  dplyr::select(i_row, text) |>
  dplyr::mutate_all(as.character) |>
  dplyr::filter(nchar(text)>50 & nchar(text)<1000) |>
  utils::head(500) |>
  tibble::deframe()

do.call(
  textreuse::TextReuseCorpus,
  list(
    text = input_text,
    tokenizer = custom_tokenize_ngrams,
    minhash_func = textreuse::minhash_generator(n = 60, seed = 123456),
    keep_tokens = TRUE,
    progress = TRUE,
    n = nx
  )
)

内容的提问来源于stack exchange，提问作者hswerdfe