大矩阵欧氏距离计算的高内存时间效率实现方案咨询
我希望找到在大矩阵上计算欧氏距离时,兼顾内存占用与运行时间效率的最优实现方式。我已完成一组小型基准测试,对比了我已知的几个R包表现:parallelDist、geodist、fields和stats;同时也参考了结合Rcpp与bigmemory的自定义实现。以下是我得到的测试结果(附可复现代码),想咨询是否还有其他高效的包或解决方案可用于该任务:
测试结果
benchmrk #> package time alloc #>1: parDist 0.298 5.369186e-04 #>2: fields 1.079 9.486198e-03 #>3: rcpp 54.422 2.161113e+00 #>4: stats 0.770 5.788603e+01 #>5: geodist 2.513 1.157635e+02 # 绘制效率对比图 ggplot(benchmrk, aes(x=alloc , y=time, color= package, label=package)) + geom_label(alpha=.5) + coord_trans(x="log10", y="log10") + theme(legend.position = "none")

可复现示例代码
library(parallelDist) library(geodist) library(fields) library(stats) library(bigmemory) library(Rcpp) library(lineprof) library(geobr) library(sf) library(ggplot2) library(data.table) # 数据输入 df <- geobr::read_weighting_area() gc(reset = T) # 转换投影为UTM df <- st_transform(df, crs = 3857) # 提取空间坐标 coords <- suppressWarnings(st_coordinates( st_centroid(df) )) # 准备自定义Rcpp函数 sourceCpp("euc_dist.cpp") bigMatrixEuc <- function(bigMat){ zeros <- big.matrix(nrow = nrow(bigMat)-1, ncol = nrow(bigMat)-1, init = 0, type = typeof(bigMat)) BigArmaEuc(bigMat@address, zeros@address) return(zeros) } ### 启动性能测试 perf_fields <- lineprof(dist_fields <- fields::rdist(coords) ) perf_geodist <- lineprof(dist_geodist <- geodist::geodist(coords, measure = "cheap") ) perf_stats <- lineprof(dist_stats <- stats::dist(coords) ) perf_parDist <- lineprof(dist_parDist <- parallelDist::parDist(coords, method = "euclidean") ) perf_rcpp <- lineprof(dist_rcpp <- bigMatrixEuc( as.big.matrix(coords) ) ) perf_fields$package <- 'fields' perf_geodist$package <- 'geodist' perf_stats$package <- 'stats' perf_parDist$package <- 'parDist' perf_rcpp$package <- 'rcpp' # 汇总结果 benchmrk <- rbind(perf_fields, perf_geodist, perf_stats , perf_parDist, perf_rcpp) benchmrk <- setDT(benchmrk)[, .(time =sum(time), alloc = sum(alloc)), by=package][order(alloc)] benchmrk
内容的提问来源于stack exchange,提问作者rafa.pereira
相关产品推荐
相关产品推荐

