从JPEG创建Spark对象并在非转换函数上使用Spark_apply()
通常,当人们想要在自定义函数上使用从JPEG创建Spark对象并在非转换函数上使用Spark_apply(),r,binary,jpeg,sparklyr,R,Binary,Jpeg,Sparklyr,通常,当人们想要在自定义函数上使用sparkyr时(例如,我只遇到一个示例,其中单个本地数据帧是copy_to()或spark_read_csv()到远程数据源,然后在其上使用spark_apply()。示例仅供说明: library(sparklyr) sc <- spark_connect(master = "local") n_sim = 100 iris_samps <- iris %>% dplyr::filter(Species == "virginica") %
sparkyr
时(例如,我只遇到一个示例,其中单个本地数据帧是copy_to()
或spark_read_csv()
到远程数据源,然后在其上使用spark_apply()
。示例仅供说明:
library(sparklyr)
sc <- spark_connect(master = "local")
n_sim = 100
iris_samps <- iris %>% dplyr::filter(Species == "virginica") %>%
sapply(rep.int, times=n_sim) %>% cbind(replicate = rep(1:n_sim, each = 50)) %>%
data.frame() %>%
dplyr::group_by(replicate) %>%
dplyr::sample_n(50, replace = TRUE)
iris_samps_tbl <- copy_to(sc, iris_samps)
iris_samps_tbl %>%
spark_apply(function(x) {mean(x$Petal_Length)},
group_by = "replicate") %>%
ggplot(aes(x = result)) + geom_histogram(bins = 20) + ggtitle("Histogram of 100 Bootstrapped Means using sparklyr")
请注意:我第一次尝试加速此代码是1)使用
profvis
查找瓶颈(即for循环)2)使用for循环上的foreach
包。这导致了较慢的代码,这表明我的并行级别太低。据我所知,sparklyr
与其说是并行计算,不如说是分布式计算,所以这可能会奏效;你到底想实现什么?我想在非翻译函数上使用SparkyR
,特别是针对composeMosaicFromImageRandom()
。你是否在单个上成功使用了spark\u apply()
?帮助说明您需要提供>一个可强制到spark数据帧的对象(通常是spark)。
function (originalImageFileName, outputImageFileName, imagesToUseInMosaic,
useGradients = FALSE, removeTiles = TRUE, fracLibSizeThreshold = 0.7,
repFracSize = 0.25, verbose = TRUE)
{
if (verbose) {
cat(paste("\n ------------------------------------------------ \n"))
cat(paste(" R Simple Mosaic composer - random version \n"))
cat(paste(" ------------------------------------------------ \n\n"))
}
if (verbose) {
cat(paste(" Creating the library... \n"))
}
libForMosaicFull <- createLibraryIndexDataFrame(imagesToUseInMosaic,
saveLibraryIndex = F, useGradients = useGradients)
libForMosaic <- libForMosaicFull
filenameArray <- list.files(imagesToUseInMosaic, full.names = TRUE)
originalImage <- jpeg::readJPEG(filenameArray[1])
xTileSize <- dim(originalImage[, , 1])[1]
yTileSize <- dim(originalImage[, , 1])[2]
if (verbose) {
cat(paste(" -- Tiles in the Library : ", length(libForMosaic[,
1]), "\n"))
cat(paste(" -- Tile dimensions : ", xTileSize, " x ",
yTileSize, "\n"))
}
if (verbose) {
cat(paste("\n"))
cat(paste(" Reading the original image... \n"))
}
originalImage <- jpeg::readJPEG(originalImageFileName)
xOrigImgSize <- dim(originalImage[, , 1])[1]
yOrigImgSize <- dim(originalImage[, , 1])[2]
if (verbose) {
cat(paste(" -- Original image dimensions : ", xOrigImgSize,
" x ", yOrigImgSize, "\n"))
cat(paste(" -- Output image dimensions : ", ((xOrigImgSize -
2) * xTileSize), " x ", ((yOrigImgSize - 2) * yTileSize),
"\n"))
}
if (verbose) {
cat(paste("\n"))
cat(paste(" Computing the mosaic... \n"))
}
outputImage <- array(dim = c(((xOrigImgSize - 2) * xTileSize),
((yOrigImgSize - 2) * yTileSize), 3))
removedList <- c()
l <- 1
pCoord <- matrix(nrow = ((xOrigImgSize - 2) * (yOrigImgSize -
2)), ncol = 2)
for (i in 2:(xOrigImgSize - 1)) {
for (j in 2:(yOrigImgSize - 1)) {
pCoord[l, 1] <- i
pCoord[l, 2] <- j
l <- l + 1
}
}
npixels <- length(pCoord[, 1])
for (i in 1:npixels) {
idx <- round(runif(1, 1, length(pCoord[, 1])))
pixelRGBandNeigArray <- computeStatisticalQuantitiesPixel(pCoord[idx,
1], pCoord[idx, 2], originalImage, useGradients)
tileFilename <- getCloseMatch(pixelRGBandNeigArray,
libForMosaic)
startI <- (pCoord[idx, 1] - 2) * xTileSize + 1
startJ <- (pCoord[idx, 2] - 2) * yTileSize + 1
outputImage[startI:(startI + xTileSize - 1), startJ:(startJ +
yTileSize - 1), ] <- jpeg::readJPEG(tileFilename)
if (removeTiles) {
libForMosaic <- removeTile(tileFilename, libForMosaic)
removedList <- c(removedList, tileFilename)
if (length(libForMosaic[, 1]) < (fracLibSizeThreshold *
length(libForMosaicFull[, 1]))) {
idxs <- runif(round(0.25 * length(libForMosaicFull[,
1])), 1, length(removedList))
for (ii in 1:length(idxs)) {
libForMosaic <- addBackTile(removedList[idxs[ii]],
libForMosaic, libForMosaicFull)
}
removedList <- removedList[-idxs]
}
}
if (length(pCoord[, 1]) > 2) {
pCoord <- pCoord[-idx, ]
}
}
if (verbose) {
cat(paste("\n"))
cat(paste(" Done!\n\n"))
}
jpeg::writeJPEG(outputImage, outputImageFileName)
}