diff --git a/docs/site/builtins-reference.md b/docs/site/builtins-reference.md index 22b335866cb..8dcc1ef61b9 100644 --- a/docs/site/builtins-reference.md +++ b/docs/site/builtins-reference.md @@ -72,6 +72,8 @@ limitations under the License. * [`outlier`-Function](#outlier-function) * [`outlierByDB`-Function](#outlierByDB-function) * [`pnmf`-Function](#pnmf-function) + * [`powerTransform`-Function](#powerTransform-function) + * [`powerTransformApply`-Function](#powerTransformApply-function) * [`scale`-Function](#scale-function) * [`setdiff`-Function](#setdiff-function) * [`sherlock`-Function](#sherlock-function) @@ -1840,6 +1842,78 @@ X = rand(rows = 50, cols = 10) [W, H] = pnmf(X = X, rnk = 2, eps = 10^-8, maxi = 10, verbose = TRUE) ``` +## `powerTransform`-Function + +The `powerTransform`-function estimates one power parameter per column and transforms the input matrix. It uses +Yeo-Johnson by default and can optionally use Box-Cox for strictly positive data. +NaN entries are preserved, while parameter estimation and standardization use the non-NaN entries in each column. + +### Usage + +```r +powerTransform(X, method="yeo-johnson", standardize=TRUE) +``` + +### Arguments + +| Name | Type | Default | Description | +| :---------- | :------------- | :-------------- | :---------- | +| X | Matrix[Double] | required | Matrix of feature vectors. | +| method | String | `"yeo-johnson"` | Transformation method: `"yeo-johnson"` or `"box-cox"`. | +| standardize | Boolean | TRUE | Whether to center and scale the transformed columns. | + +### Returns + +| Type | Description | +| :------------- | :---------- | +| Matrix[Double] | Transformed matrix. | +| Matrix[Double] | Row vector of estimated power parameters. | +| Matrix[Double] | Row vector of transformed column means, or an empty matrix when standardization is disabled. | +| Matrix[Double] | Row vector of transformed column scales, or an empty matrix when standardization is disabled. | + +### Example + +```r +X = matrix("-2 -1 0 1 2 4", rows=6, cols=1) +[Y, lambdas, means, scales] = powerTransform(X=X) +``` + +## `powerTransformApply`-Function + +The `powerTransformApply`-function transforms a matrix using parameters previously returned by `powerTransform`. +NaN entries are preserved in the transformed matrix. + +### Usage + +```r +powerTransformApply(X, lambdas, means, scales, method="yeo-johnson") +``` + +### Arguments + +| Name | Type | Default | Description | +| :------ | :------------- | :--------------- | :---------- | +| X | Matrix[Double] | required | Matrix of feature vectors. | +| lambdas | Matrix[Double] | required | Row vector of fitted power parameters. | +| means | Matrix[Double] | required | Row vector of fitted means, or an empty matrix to skip standardization. | +| scales | Matrix[Double] | required | Row vector of fitted scales, or an empty matrix to skip standardization. | +| method | String | `"yeo-johnson"` | Transformation method used during fitting. | + +### Returns + +| Type | Description | +| :------------- | :---------- | +| Matrix[Double] | Transformed matrix. | + +### Example + +```r +X = matrix("-2 -1 0 1 2 4", rows=6, cols=1) +[Y, lambdas, means, scales] = powerTransform(X=X) +Xnew = matrix("-3 0 3", rows=3, cols=1) +Ynew = powerTransformApply(X=Xnew, lambdas=lambdas, means=means, scales=scales) +``` + ## `scale`-Function diff --git a/scripts/builtin/powerTransform.dml b/scripts/builtin/powerTransform.dml new file mode 100644 index 00000000000..1e7cfab3627 --- /dev/null +++ b/scripts/builtin/powerTransform.dml @@ -0,0 +1,383 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Power transformation using the selected method. +# Reduces feature skewness by estimating and applying an optimal transformation parameter for each column. +# +# INPUT: +# ------------------------------------------------------------------------------------- +# X Input feature matrix of shape n-by-m +# method Power transformation method: "yeo-johnson" (default) or "box-cox" +# standardize Whether to normalize transformed columns to zero mean and unit variance +# ------------------------------------------------------------------------------------- +# +# OUTPUT: +# ------------------------------------------------------------------------------------- +# Y Power-transformed matrix of shape n-by-m +# lambdas Estimated lambda parameters of shape 1-by-m, one per column +# means Transformed column means of shape 1-by-m, or an empty matrix when not standardized +# scales Transformed column scales of shape 1-by-m, or an empty matrix when not standardized +# ------------------------------------------------------------------------------------- + +m_powerTransform = function( + Matrix[Double] X, + String method="yeo-johnson", + Boolean standardize=TRUE) + return ( + Matrix[Double] Y, + Matrix[Double] lambdas, + Matrix[Double] means, + Matrix[Double] scales) +{ + if (method != "yeo-johnson" & method != "box-cox") { + stop("powerTransform: unsupported method '" + method + + "'; expected 'yeo-johnson' or 'box-cox'") + } + + validatedX = replace(target=X, pattern=NaN, replacement=1.0) + if (method == "box-cox" & min(validatedX) <= 0.0) { + stop("powerTransform: Box-Cox requires strictly positive input") + } + + m = ncol(X) + lambdas = matrix(1.0, rows=1, cols=m) # Initialize first, then replace each column with the best lambdas + + # Estimate lambda for each column separately + for (j in 1:m){ + x = X[,j] + xObserved = removeEmpty(target=x, margin="rows", select=(is.na(x) == 0)) + observedN = nrow(xObserved) + + # Yeo-Johnson leaves constant columns unchanged; Box-Cox rejects them + if (observedN == 0) { + lambdas[1,j] = 1.0 + } + else if (max(xObserved) == min(xObserved)) { + if (method == "yeo-johnson") { + lambdas[1,j] = 1.0; + } + else { + stop("powerTransform: Box-Cox does not support constant columns") + } + } + else{ + lambdas[1,j] = ptEstimateLambda(xObserved, method); + } + } + + # Apply the fitted transformation before optional standardization + emptyStats = matrix(0.0, rows=0, cols=0) + Y = powerTransformApply(X, lambdas, emptyStats, emptyStats, method); + + means = matrix(0.0, rows=0, cols=0) + scales = matrix(0.0, rows=0, cols=0) + + if (standardize) { + means = matrix(0.0, rows=1, cols=m) + scales = matrix(1.0, rows=1, cols=m) + + for (j in 1:m) { + y = Y[,j] + yObserved = removeEmpty(target=y, margin="rows", select=(is.na(y) == 0)) + observedN = nrow(yObserved) + + if (observedN > 0) { + means[1,j] = mean(yObserved) + scale = sqrt(sum((yObserved - means[1,j])^2) / observedN) + if (!is.na(scale) & !is.infinite(scale) & scale != 0.0) { + scales[1,j] = scale + } + } + + Y[,j] = ifelse(is.na(y), NaN, (y - means[1,j]) / scales[1,j]) + } + } +} +ptEstimateLambda = function(Matrix[Double] x, String method) + return (Double lambda) +{ + lower = -2.0; + upper = 2.0; + + if (method == "box-cox") { + jacTerm = sum(log(x)) + } + else { + jacTerm = sum(sign(x) * log(abs(x) + 1.0)) + } + + lambda = ptBrentSearch(x, lower, upper, method, jacTerm); +} + +# Compute negative log likelihood; lower lambda score is better + +ptNegLogLikelihood = function( + Matrix[Double] x, + Double lambda, + String method, + Double jacTerm) + return (Double negLogLikelihood) +{ + eps = 1e-12 + if (method == "box-cox") { + if (abs(lambda) < eps) { + y = log(x) + } + else { + y = (x^lambda - 1.0) / lambda + } + } + else { + nonnegative = x >= 0 + xPos = ifelse(nonnegative, x, 0.0) + xNeg = ifelse(nonnegative, 0.0, x) + + if (abs(lambda) < eps) { + yPos = log(xPos + 1.0) + } + else { + yPos = ((xPos + 1.0)^lambda - 1.0) / lambda + } + + if (abs(lambda - 2.0) < eps) { + yNeg = -log(1.0 - xNeg) + } + else { + yNeg = -((1.0 - xNeg)^(2.0 - lambda) - 1.0) / (2.0 - lambda) + } + + y = ifelse(nonnegative, yPos, yNeg) + } + + n = nrow(x); + yMean = mean(y); + yVariance = sum((y - yMean)^2) / n; + + if (sum(is.na(y)) > 0 | sum(is.infinite(y)) > 0 | + is.na(yVariance) | is.infinite(yVariance) | yVariance <= 0.0) { + negLogLikelihood = 1e300 + } + else { + logLikelihood = -n / 2.0 * log(yVariance) + (lambda - 1.0) * jacTerm; + negLogLikelihood = -logLikelihood; + if (is.na(negLogLikelihood) | is.infinite(negLogLikelihood)) { + negLogLikelihood = 1e300 + } + } +} + +# Minimize the negative log likelihood with Brent optimization +ptBrentSearch = function( + Matrix[Double] x, + Double lower, + Double upper, + String method, + Double jacTerm) + return (Double lambdaOptimal) +{ + # Expand the initial interval until it brackets a minimum + goldenRatio = 1.618034; + maxBracketIterations = 1000; + lowerScore = ptNegLogLikelihood(x, lower, method, jacTerm); + upperScore = ptNegLogLikelihood(x, upper, method, jacTerm); + + lambdaOptimal = 1.0 + bestScore = ptNegLogLikelihood(x, lambdaOptimal, method, jacTerm) + if (lowerScore < bestScore) { + lambdaOptimal = lower + bestScore = lowerScore + } + if (upperScore < bestScore) { + lambdaOptimal = upper + bestScore = upperScore + } + + if (lowerScore < upperScore) { + xa = upper; + fa = upperScore; + xb = lower; + fb = lowerScore; + } + else { + xa = lower; + fa = lowerScore; + xb = upper; + fb = upperScore; + } + + initialXc = xb + goldenRatio * (xb - xa); + initialFc = ptNegLogLikelihood(x, initialXc, method, jacTerm); + xc = initialXc; + fc = initialFc; + if (fc < bestScore) { + lambdaOptimal = xc + bestScore = fc + } + bracketIteration = 0; + while ((fc < fb) & (bracketIteration < maxBracketIterations)) { + nextXc = xc + goldenRatio * (xc - xb); + nextFc = ptNegLogLikelihood(x, nextXc, method, jacTerm); + xa = xb; + fa = fb; + xb = xc; + fb = fc; + xc = nextXc; + fc = nextFc; + if (fc < bestScore) { + lambdaOptimal = xc + bestScore = fc + } + bracketIteration = bracketIteration + 1; + } + + validBracket = bracketIteration < maxBracketIterations & + (((fb < fa) & (fb <= fc)) | ((fb <= fa) & (fb < fc))) + + if (validBracket) { + a = min(xa, xc); + b = max(xa, xc); + + goldenMean = 0.3819660112501051; + sqrtEpsilon = sqrt(2.2e-16); + tolerance = 1.48e-8; + maxIterations = 500; + + xf = a + goldenMean * (b - a); + nfc = xf; + fulc = xf; + fx = ptNegLogLikelihood(x, xf, method, jacTerm); + fnfc = fx; + ffulc = fx; + if (fx < bestScore) { + lambdaOptimal = xf + bestScore = fx + } + + rat = 0.0; + e = 0.0; + midpoint = 0.5 * (a + b); + tol1 = sqrtEpsilon * abs(xf) + tolerance / 3.0; + tol2 = 2.0 * tol1; + + iteration = 0; + while ((abs(xf - midpoint) > (tol2 - 0.5 * (b - a))) & + (iteration < maxIterations)) { + goldenStep = TRUE; + + if (abs(e) > tol1) { + goldenStep = FALSE; + r = (xf - nfc) * (fx - ffulc); + q = (xf - fulc) * (fx - fnfc); + p = (xf - fulc) * q - (xf - nfc) * r; + q = 2.0 * (q - r); + + if (q > 0.0) { + p = -p; + } + + q = abs(q); + previousE = e; + e = rat; + + if ((q > 0.0) & (abs(p) < abs(0.5 * q * previousE)) & + (p > q * (a - xf)) & (p < q * (b - xf))) { + rat = p / q; + candidate = xf + rat; + + if (((candidate - a) < tol2) | ((b - candidate) < tol2)) { + if (midpoint >= xf) { + rat = tol1; + } + else { + rat = -tol1; + } + } + } + else { + goldenStep = TRUE; + } + } + + if (goldenStep) { + if (xf >= midpoint) { + e = a - xf; + } + else { + e = b - xf; + } + rat = goldenMean * e; + } + + if (rat >= 0.0) { + candidate = xf + max(abs(rat), tol1); + } + else { + candidate = xf - max(abs(rat), tol1); + } + + fCandidate = ptNegLogLikelihood(x, candidate, method, jacTerm); + if (fCandidate < bestScore) { + lambdaOptimal = candidate + bestScore = fCandidate + } + + if (fCandidate <= fx) { + if (candidate >= xf) { + a = xf; + } + else { + b = xf; + } + + fulc = nfc; + ffulc = fnfc; + nfc = xf; + fnfc = fx; + xf = candidate; + fx = fCandidate; + } + else { + if (candidate < xf) { + a = candidate; + } + else { + b = candidate; + } + + if ((fCandidate <= fnfc) | (nfc == xf)) { + fulc = nfc; + ffulc = fnfc; + nfc = candidate; + fnfc = fCandidate; + } + else if ((fCandidate <= ffulc) | (fulc == xf) | (fulc == nfc)) { + fulc = candidate; + ffulc = fCandidate; + } + } + + midpoint = 0.5 * (a + b); + tol1 = sqrtEpsilon * abs(xf) + tolerance / 3.0; + tol2 = 2.0 * tol1; + iteration = iteration + 1; + } + } +} diff --git a/scripts/builtin/powerTransformApply.dml b/scripts/builtin/powerTransformApply.dml new file mode 100644 index 00000000000..3faf4460a86 --- /dev/null +++ b/scripts/builtin/powerTransformApply.dml @@ -0,0 +1,131 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Applies a fitted power transformation and optional standardization. +# Transforms each feature using its previously estimated lambda and scaling parameters. +# +# INPUT: +# ------------------------------------------------------------------------------------- +# X Input feature matrix of shape n-by-m +# lambdas Precomputed lambda parameters of shape 1-by-m, one per column +# means Transformed column means of shape 1-by-m; empty to skip standardization +# scales Transformed column scales of shape 1-by-m; empty to skip standardization +# method Power transformation method: "yeo-johnson" (default) or "box-cox" +# ------------------------------------------------------------------------------------- +# +# OUTPUT: +# ------------------------------------------------------------------------------------- +# Y Power-transformed matrix of shape n-by-m +# ------------------------------------------------------------------------------------- + + +m_powerTransformApply = function( + Matrix[Double] X, + Matrix[Double] lambdas, + Matrix[Double] means, + Matrix[Double] scales, + String method="yeo-johnson") + return (Matrix[Double] Y) +{ + n = nrow(X) + m = ncol(X) + + if (method != "yeo-johnson" & method != "box-cox") { + stop("powerTransformApply: unsupported method '" + method + + "'; expected 'yeo-johnson' or 'box-cox'") + } + + validatedX = replace(target=X, pattern=NaN, replacement=1.0) + if (method == "box-cox" & min(validatedX) <= 0.0) { + stop("powerTransformApply: Box-Cox requires strictly positive input") + } + + if (nrow(lambdas) != 1 | ncol(lambdas) != m) { + stop("powerTransformApply: lambdas must have shape 1-by-ncol(X)") + } + + hasMeans = nrow(means) > 0 | ncol(means) > 0 + hasScales = nrow(scales) > 0 | ncol(scales) > 0 + + if (hasMeans != hasScales) { + stop("powerTransformApply: means and scales must either both be provided or both be empty") + } + + if (hasMeans & (nrow(means) != 1 | ncol(means) != m | + nrow(scales) != 1 | ncol(scales) != m)) { + stop("powerTransformApply: means and scales must have shape 1-by-ncol(X)") + } + + Y = matrix(0.0, rows=n, cols=m) + + # Handle boundary points (0 and 2) + eps = 1e-12 + + # Loop over columns for transformation + for (j in 1:m){ + x = X[,j] + nanMask = is.na(x) + lambda_j = as.scalar(lambdas[1,j]) + + if (method == "box-cox") { + x = replace(target=x, pattern=NaN, replacement=1.0) + if (abs(lambda_j) < eps) { + y = log(x) + } + else { + y = (x^lambda_j - 1.0) / lambda_j + } + } + else { + x = replace(target=x, pattern=NaN, replacement=0.0) + nonnegative = x >= 0 + + # Use intermediate inputs to avoid invalid domains in scoring or fractional powers + x_pos = ifelse(nonnegative, x, 0.0) + x_neg = ifelse(nonnegative, 0.0, x) + + # Transform nonnegative values (x>=0) + if (abs(lambda_j) < eps) { + y_pos = log(x_pos + 1) + } + else { + y_pos = ((x_pos + 1)^lambda_j - 1) / lambda_j + } + + # Transform negative values (x<0) + if (abs(lambda_j - 2) < eps) { + y_neg = -log(1 - x_neg) + } + else { + y_neg = -((1 - x_neg)^(2 - lambda_j) - 1) / (2 - lambda_j) + } + + # Combine the two branches + y = ifelse(nonnegative, y_pos, y_neg) + } + + Y[,j] = ifelse(nanMask, NaN, y) + } + + if (hasMeans) { + Y = (Y - means) / scales + } +} diff --git a/src/main/java/org/apache/sysds/common/Builtins.java b/src/main/java/org/apache/sysds/common/Builtins.java index f5719641df7..5ea55507193 100644 --- a/src/main/java/org/apache/sysds/common/Builtins.java +++ b/src/main/java/org/apache/sysds/common/Builtins.java @@ -89,280 +89,104 @@ public enum Builtins { COLVAR("colVars", false), COMPONENTS("components", true), COMPRESS("compress", false, ReturnType.MULTI_RETURN), - QUANTIZE_COMPRESS("quantize_compress", false, ReturnType.MULTI_RETURN), - CONFUSIONMATRIX("confusionMatrix", true), - CONV2D("conv2d", false), - CONV2D_BACKWARD_FILTER("conv2d_backward_filter", false), - CONV2D_BACKWARD_DATA("conv2d_backward_data", false), - COOCCURRENCEMATRIX("cooccurrenceMatrix", true), - COR("cor", true), - CORRECTTYPOS("correctTypos", true), - CORRECTTYPOSAPPLY("correctTyposApply", true), - COS("cos", false), - COSH("cosh", false), - COV("cov", false), - COX("cox", true), - CSPLINE("cspline", true), - CSPLINE_CG("csplineCG", true), - CSPLINE_DS("csplineDS", true), - CUMMAX("cummax", false), - CUMMIN("cummin", false), - CUMPROD("cumprod", false), - CUMSUM("cumsum", false), - CUMSUMPROD("cumsumprod", false), - DBSCAN("dbscan", true), - DBSCANAPPLY("dbscanApply", true), - DECISIONTREE("decisionTree", true), - DECISIONTREEPREDICT("decisionTreePredict", true), - DECOMPRESS("decompress", false), - DEDUP("dedup", true), - DEEPWALK("deepWalk", true), - DET("det", false), - DETECTSCHEMA("detectSchema", false), - DENIALCONSTRAINTS("denialConstraints", true), - DIFFERENCESTATISTICS("differenceStatistics", true), - DIAG("diag", false), - DISCOVER_FD("discoverFD", true), - DISCOVER_MD("mdedup", true), - SETDIFF("setdiff", true), - DIST("dist", true), - DMV("dmv", true), - DROP_INVALID_TYPE("dropInvalidType", false), - DROP_INVALID_LENGTH("dropInvalidLength", false), - EIGEN("eigen", false, ReturnType.MULTI_RETURN), - EMA("ema", true), - EXISTS("exists", false), - EXECUTE_PIPELINE("executePipeline", true), - EXP("exp", false), - EVAL("eval", false), - EVALLIST("evalList", false), - F1SCORE("f1Score", true), - FDR("fdr", "FDR", true), - FIT_PIPELINE("fit_pipeline", true), - FIX_INVALID_LENGTHS("fixInvalidLengths", true), - FIX_INVALID_LENGTHS_APPLY("fixInvalidLengthsApply", true), - FFT("fft", false, ReturnType.MULTI_RETURN), - FFT_LINEARIZED("fft_linearized", false, ReturnType.MULTI_RETURN), - FF_TRAIN("ffTrain", true), - FF_PREDICT("ffPredict", true), - FLOOR("floor", false), - FLATTENQUANTILE("flattenQuantile", true), - FRAME_SORT("frameSort", true), - FRAME_ROW_REPLICATE("freplicate", false), - FREQUENCYENCODE("frequencyEncode", true), - FREQUENCYENCODEAPPLY("frequencyEncodeApply", true), - GARCH("garch", true), - GAUSSIAN_CLASSIFIER("gaussianClassifier", true), - GET_ACCURACY("getAccuracy", true), - GET_CATEGORICAL_MASK("getCategoricalMask", false), - GLM("glm", true), - GLM_PREDICT("glmPredict", true), - GLOVE("glove", true), - GMM("gmm", true), - GMM_PREDICT("gmmPredict", true), - GNMF("gnmf", true), - GRID_SEARCH("gridSearch", true), - TOPK_CLEANING("topk_cleaning", true), - HOSPITAL_RESIDENCY_MATCH("hospitalResidencyMatch", true), - HYPERBAND("hyperband", true), - IFELSE("ifelse", false), - IFFT("ifft", false, ReturnType.MULTI_RETURN), - IFFT_LINEARIZED("ifft_linearized", false, ReturnType.MULTI_RETURN), - IMG_MIRROR("img_mirror", true), - IMG_MIRROR_LINEARIZED("img_mirror_linearized", true), - IMG_BRIGHTNESS("img_brightness", true), - IMG_BRIGHTNESS_LINEARIZED("img_brightness_linearized", true), - IMG_CROP("img_crop", true), - IMG_CROP_LINEARIZED("img_crop_linearized", true), - IMG_TRANSFORM("img_transform", true), - IMG_TRANSFORM_LINEARIZED("img_transform_linearized", true), - IMG_TRANSLATE("img_translate", true), - IMG_TRANSLATE_LINEARIZED("img_translate_linearized", true), - IMG_ROTATE("img_rotate", true), - IMG_ROTATE_LINEARIZED("img_rotate_linearized", true), - IMG_SHEAR("img_shear", true), - IMG_SHEAR_LINEARIZED("img_shear_linearized", true), - IMG_CUTOUT("img_cutout", true), - IMG_CUTOUT_LINEARIZED("img_cutout_linearized", true), - IMG_SAMPLE_PAIRING("img_sample_pairing", true), - IMG_SAMPLE_PAIRING_LINEARIZED("img_sample_pairing_linearized", true), - IMG_INVERT("img_invert", true), - IMG_INVERT_LINEARIZED("img_invert_linearized", true), - IMG_POSTERIZE("img_posterize", true), - IMG_POSTERIZE_LINEARIZED("img_posterize_linearized", true), - IMPURITY_MEASURES("impurityMeasures", true), - IMPUTE_BY_KNN("imputeByKNN", true), - IMPUTE_BY_MEAN("imputeByMean", true), - IMPUTE_BY_MEAN_APPLY("imputeByMeanApply", true), - IMPUTE_BY_MEDIAN("imputeByMedian", true), - IMPUTE_BY_MEDIAN_APPLY("imputeByMedianApply", true), - IMPUTE_BY_MODE("imputeByMode", true), - IMPUTE_BY_MODE_APPLY("imputeByModeApply", true), - IMPUTE_FD("imputeByFD", true), - IMPUTE_FD_APPLY("imputeByFDApply", true), - INCSLICELINE("incSliceLine", true), - INTERQUANTILE("interQuantile", false), - INTERSECT("intersect", true), - INVERSE("inv", "inverse", false), - IQM("interQuartileMean", false), - ISNA("is.na", "isNA", false), - ISNAN("is.nan", "isNaN", false), - ISINF("is.infinite", "isInf", false), - ISN_TRAIN("independentSubnetTrain", true), - KM("km", true), - KMEANS("kmeans", true), - KMEANSPREDICT("kmeansPredict", true), - KNNBF("knnbf", true), - KNNGRAPH("knnGraph", true), - KNN("knn", true), - L2SVM("l2svm", true), - L2SVMPREDICT("l2svmPredict", true), - LASSO("lasso", true), - LENET_TRAIN("lenetTrain", true), - LENET_PREDICT("lenetPredict", true), - LENGTH("length", false), - LINEAGE("lineage", false), - LIST("list", false), //note: builtin and parbuiltin - LM("lm", true), - LMCG("lmCG", true), - LMDS("lmDS", true), - LMPREDICT("lmPredict", true), - LMPREDICT_STATS("lmPredictStats", true), - LOCAL("local", false), - LOG("log", false), - LOGSUMEXP("logSumExp", true), - LSTM("lstm", false, ReturnType.MULTI_RETURN), - LSTM_BACKWARD("lstm_backward", false, ReturnType.MULTI_RETURN), - LU("lu", false, ReturnType.MULTI_RETURN), - MAP("map", false), - MATRIXPROFILE("matrixProfile", true), - MAX("max", "pmax", false), - MAX_POOL("max_pool", false), - MAX_POOL_BACKWARD("max_pool_backward", false), - MCC("mcc", true), - MAE("mae", true), - MAPE("mape", true), - MEAN("mean", "avg", false), - MEDIAN("median", false), - MICE("mice", true), - MICE_APPLY("miceApply", true), - MIN("min", "pmin", false), - MOMENT("moment", "centralMoment", false), - MSE("mse", true), - MSMAPE("msmape", true), - MSVM("msvm", true), - MSVMPREDICT("msvmPredict", true), - MULTILOGREG("multiLogReg", true), - MULTILOGREGPREDICT("multiLogRegPredict", true), - NA_LOCF("na_locf", true), - NAIVEBAYES("naiveBayes", true, false), - NAIVEBAYESPREDICT("naiveBayesPredict", true, false), - NCOL("ncol", false), - NORMALIZE("normalize", true), - NORMALIZEAPPLY("normalizeApply", true), - NROW("nrow", false), - NRMSE("nrmse", true), - OUTER("outer", false), - OUTLIER("outlier", true, false), //TODO parameterize opposite - OUTLIER_ARIMA("outlierByArima",true), - OUTLIER_IQR("outlierByIQR", true), - OUTLIER_IQR_APPLY("outlierByIQRApply", true), - OUTLIER_SD("outlierBySd", true), - OUTLIER_SD_APPLY("outlierBySdApply", true), - PAGERANK("pageRank", true), - PCA("pca", true), - PCAINVERSE("pcaInverse", true), - PCATRANSFORM("pcaTransform", true), - PNMF("pnmf", true), - PPCA("ppca", true), - PPRED("ppred", false), - PROD("prod", false), - PSNR("psnr", true), - QR("qr", false, ReturnType.MULTI_RETURN), - QUANTILE("quantile", false), - QUANTIZEBYCLUSTER("quantizeByCluster", true), - RANDOM_FOREST("randomForest", true), - RANDOM_FOREST_PREDICT("randomForestPredict", true), - RANGE("range", false), - RAGROUPBY("raGroupby", true), - RAJOIN("raJoin", true), - RASELECTION("raSelection", true), - RBIND("rbind", false), - RCM("rowClassMeet", "rcm", false, false, ReturnType.MULTI_RETURN), - REMOVE("remove", false, ReturnType.MULTI_RETURN), - REV("rev", false), - ROLL("roll", false), - ROUND("round", false), - ROW_COUNT_DISTINCT("rowCountDistinct",false), - ROWCUMSUM("rowcumsum", false), - ROWINDEXMAX("rowIndexMax", false), - ROWINDEXMIN("rowIndexMin", false), - ROWMAX("rowMaxs", false), - ROWMEAN("rowMeans", false), - ROWMIN("rowMins", false), - ROWPROD("rowProds", false), - ROWSD("rowSds", false), - ROWSUM("rowSums", false), - ROWVAR("rowVars", false), - RMSE("rmse", true), - SAMPLE("sample", false), - SD("sd", false), - SELVARTHRESH("selectByVarThresh", true), - SEQ("seq", false), - SES("ses", true), - SYMMETRICDIFFERENCE("symmetricDifference", true), - SHAPEXPLAINER("shapExplainer", true), - SHERLOCK("sherlock", true), - SHERLOCKPREDICT("sherlockPredict", true), - SHORTESTPATH("shortestPath", true), - SIGMOID("sigmoid", true), // 1 / (1 + exp(-X)) - SIGN("sign", false), - SIN("sin", false), - SINH("sinh", false), - SLICEFINDER("slicefinder", true), //TODO remove - SLICELINE("sliceLine", true), - SLICELINE_DEBUG("sliceLineDebug", true), - SLICELINE_EXTRACT("sliceLineExtract", true), - SKEWNESS("skewness", true), - SMAPE("smape", true), - SMOTE("smote", true), - SOFTMAX("softmax", true), - SOLVE("solve", false), - SPLIT("split", true), - SPLIT_BALANCED("splitBalanced", true), - STABLE_MARRIAGE("stableMarriage", true), - STATSNA("statsNA", true), - STRATSTATS("stratstats", true), - STEPLM("steplm",true, ReturnType.MULTI_RETURN), - STFT("stft", false, ReturnType.MULTI_RETURN), - SQRT("sqrt", false), - SQRT_MATRIX("sqrtMatrix", true), - SQRT_MATRIX_JAVA("sqrtMatrixJava", false, ReturnType.SINGLE_RETURN), - SUM("sum", false), - SVD("svd", false, ReturnType.MULTI_RETURN), - TABLE("table", "ctable", false), - TAN("tan", false), - TANH("tanh", false), - TO_ONE_HOT("toOneHot", true), - TOMEKLINK("tomeklink", true), - TRACE("trace", "tr", false), - TRANS("t", false), - TSNE("tSNE", true), - TYPEOF("typeof", false), - UNIVAR("univar", true), - UNION("union", true), - VAR("var", false), - VALUE_SWAP("valueSwap", false), - VECTOR_TO_CSV("vectorToCsv", true), - WER("wer", true, false), - WINSORIZE("winsorize", true, false), //TODO parameterize w/ prob, min/max val - WINSORIZEAPPLY("winsorizeApply", true, false), //TODO parameterize w/ prob, min/max val - WOE("WoE", true ), - WOEAPPLY("WoEApply", true ), - XGBOOST("xgboost", true), - XGBOOSTPREDICT("xgboostPredict", true), - XGBOOSTPREDICT_REG("xgboostPredictRegression", true), - XGBOOSTPREDICT_CLASS("xgboostPredictClassification", true), + QUANTIZE_COMPRESS("quantize_compress", false, ReturnType.MULTI_RETURN), CONFUSIONMATRIX("confusionMatrix", true), + CONV2D("conv2d", false), CONV2D_BACKWARD_FILTER("conv2d_backward_filter", false), + CONV2D_BACKWARD_DATA("conv2d_backward_data", false), COOCCURRENCEMATRIX("cooccurrenceMatrix", true), + COR("cor", true), CORRECTTYPOS("correctTypos", true), CORRECTTYPOSAPPLY("correctTyposApply", true), + COS("cos", false), COSH("cosh", false), COV("cov", false), COX("cox", true), CSPLINE("cspline", true), + CSPLINE_CG("csplineCG", true), CSPLINE_DS("csplineDS", true), CUMMAX("cummax", false), CUMMIN("cummin", false), + CUMPROD("cumprod", false), CUMSUM("cumsum", false), CUMSUMPROD("cumsumprod", false), DBSCAN("dbscan", true), + DBSCANAPPLY("dbscanApply", true), DECISIONTREE("decisionTree", true), + DECISIONTREEPREDICT("decisionTreePredict", true), DECOMPRESS("decompress", false), DEDUP("dedup", true), + DEEPWALK("deepWalk", true), DET("det", false), DETECTSCHEMA("detectSchema", false), + DENIALCONSTRAINTS("denialConstraints", true), DIFFERENCESTATISTICS("differenceStatistics", true), + DIAG("diag", false), DISCOVER_FD("discoverFD", true), DISCOVER_MD("mdedup", true), SETDIFF("setdiff", true), + DIST("dist", true), DMV("dmv", true), DROP_INVALID_TYPE("dropInvalidType", false), + DROP_INVALID_LENGTH("dropInvalidLength", false), EIGEN("eigen", false, ReturnType.MULTI_RETURN), EMA("ema", true), + EXISTS("exists", false), EXECUTE_PIPELINE("executePipeline", true), EXP("exp", false), EVAL("eval", false), + EVALLIST("evalList", false), F1SCORE("f1Score", true), FDR("fdr", "FDR", true), FIT_PIPELINE("fit_pipeline", true), + FIX_INVALID_LENGTHS("fixInvalidLengths", true), FIX_INVALID_LENGTHS_APPLY("fixInvalidLengthsApply", true), + FFT("fft", false, ReturnType.MULTI_RETURN), FFT_LINEARIZED("fft_linearized", false, ReturnType.MULTI_RETURN), + FF_TRAIN("ffTrain", true), FF_PREDICT("ffPredict", true), FLOOR("floor", false), + FLATTENQUANTILE("flattenQuantile", true), FRAME_SORT("frameSort", true), FRAME_ROW_REPLICATE("freplicate", false), + FREQUENCYENCODE("frequencyEncode", true), FREQUENCYENCODEAPPLY("frequencyEncodeApply", true), GARCH("garch", true), + GAUSSIAN_CLASSIFIER("gaussianClassifier", true), GET_ACCURACY("getAccuracy", true), + GET_CATEGORICAL_MASK("getCategoricalMask", false), GLM("glm", true), GLM_PREDICT("glmPredict", true), + GLOVE("glove", true), GMM("gmm", true), GMM_PREDICT("gmmPredict", true), GNMF("gnmf", true), + GRID_SEARCH("gridSearch", true), TOPK_CLEANING("topk_cleaning", true), + HOSPITAL_RESIDENCY_MATCH("hospitalResidencyMatch", true), HYPERBAND("hyperband", true), IFELSE("ifelse", false), + IFFT("ifft", false, ReturnType.MULTI_RETURN), IFFT_LINEARIZED("ifft_linearized", false, ReturnType.MULTI_RETURN), + IMG_MIRROR("img_mirror", true), IMG_MIRROR_LINEARIZED("img_mirror_linearized", true), + IMG_BRIGHTNESS("img_brightness", true), IMG_BRIGHTNESS_LINEARIZED("img_brightness_linearized", true), + IMG_CROP("img_crop", true), IMG_CROP_LINEARIZED("img_crop_linearized", true), IMG_TRANSFORM("img_transform", true), + IMG_TRANSFORM_LINEARIZED("img_transform_linearized", true), IMG_TRANSLATE("img_translate", true), + IMG_TRANSLATE_LINEARIZED("img_translate_linearized", true), IMG_ROTATE("img_rotate", true), + IMG_ROTATE_LINEARIZED("img_rotate_linearized", true), IMG_SHEAR("img_shear", true), + IMG_SHEAR_LINEARIZED("img_shear_linearized", true), IMG_CUTOUT("img_cutout", true), + IMG_CUTOUT_LINEARIZED("img_cutout_linearized", true), IMG_SAMPLE_PAIRING("img_sample_pairing", true), + IMG_SAMPLE_PAIRING_LINEARIZED("img_sample_pairing_linearized", true), IMG_INVERT("img_invert", true), + IMG_INVERT_LINEARIZED("img_invert_linearized", true), IMG_POSTERIZE("img_posterize", true), + IMG_POSTERIZE_LINEARIZED("img_posterize_linearized", true), IMPURITY_MEASURES("impurityMeasures", true), + IMPUTE_BY_KNN("imputeByKNN", true), IMPUTE_BY_MEAN("imputeByMean", true), + IMPUTE_BY_MEAN_APPLY("imputeByMeanApply", true), IMPUTE_BY_MEDIAN("imputeByMedian", true), + IMPUTE_BY_MEDIAN_APPLY("imputeByMedianApply", true), IMPUTE_BY_MODE("imputeByMode", true), + IMPUTE_BY_MODE_APPLY("imputeByModeApply", true), IMPUTE_FD("imputeByFD", true), + IMPUTE_FD_APPLY("imputeByFDApply", true), INCSLICELINE("incSliceLine", true), INTERQUANTILE("interQuantile", false), + INTERSECT("intersect", true), INVERSE("inv", "inverse", false), IQM("interQuartileMean", false), + ISNA("is.na", "isNA", false), ISNAN("is.nan", "isNaN", false), ISINF("is.infinite", "isInf", false), + ISN_TRAIN("independentSubnetTrain", true), KM("km", true), KMEANS("kmeans", true), + KMEANSPREDICT("kmeansPredict", true), KNNBF("knnbf", true), KNNGRAPH("knnGraph", true), KNN("knn", true), + L2SVM("l2svm", true), L2SVMPREDICT("l2svmPredict", true), LASSO("lasso", true), LENET_TRAIN("lenetTrain", true), + LENET_PREDICT("lenetPredict", true), LENGTH("length", false), LINEAGE("lineage", false), LIST("list", false), // note: + // builtin + // and + // parbuiltin + LM("lm", true), LMCG("lmCG", true), LMDS("lmDS", true), LMPREDICT("lmPredict", true), + LMPREDICT_STATS("lmPredictStats", true), LOCAL("local", false), LOG("log", false), LOGSUMEXP("logSumExp", true), + LSTM("lstm", false, ReturnType.MULTI_RETURN), LSTM_BACKWARD("lstm_backward", false, ReturnType.MULTI_RETURN), + LU("lu", false, ReturnType.MULTI_RETURN), MAP("map", false), MATRIXPROFILE("matrixProfile", true), + MAX("max", "pmax", false), MAX_POOL("max_pool", false), MAX_POOL_BACKWARD("max_pool_backward", false), + MCC("mcc", true), MAE("mae", true), MAPE("mape", true), MEAN("mean", "avg", false), MEDIAN("median", false), + MICE("mice", true), MICE_APPLY("miceApply", true), MIN("min", "pmin", false), + MOMENT("moment", "centralMoment", false), MSE("mse", true), MSMAPE("msmape", true), MSVM("msvm", true), + MSVMPREDICT("msvmPredict", true), MULTILOGREG("multiLogReg", true), MULTILOGREGPREDICT("multiLogRegPredict", true), + NA_LOCF("na_locf", true), NAIVEBAYES("naiveBayes", true, false), + NAIVEBAYESPREDICT("naiveBayesPredict", true, false), NCOL("ncol", false), NORMALIZE("normalize", true), + NORMALIZEAPPLY("normalizeApply", true), NROW("nrow", false), NRMSE("nrmse", true), OUTER("outer", false), + OUTLIER("outlier", true, false), // TODO parameterize opposite + OUTLIER_ARIMA("outlierByArima", true), OUTLIER_IQR("outlierByIQR", true), + OUTLIER_IQR_APPLY("outlierByIQRApply", true), OUTLIER_SD("outlierBySd", true), + OUTLIER_SD_APPLY("outlierBySdApply", true), PAGERANK("pageRank", true), PCA("pca", true), + PCAINVERSE("pcaInverse", true), PCATRANSFORM("pcaTransform", true), PNMF("pnmf", true), PPCA("ppca", true), + POWERTRANSFORM("powerTransform", true), POWERTRANSFORMAPPLY("powerTransformApply", true), PPRED("ppred", false), + PROD("prod", false), PSNR("psnr", true), QR("qr", false, ReturnType.MULTI_RETURN), QUANTILE("quantile", false), + QUANTIZEBYCLUSTER("quantizeByCluster", true), RANDOM_FOREST("randomForest", true), + RANDOM_FOREST_PREDICT("randomForestPredict", true), RANGE("range", false), RAGROUPBY("raGroupby", true), + RAJOIN("raJoin", true), RASELECTION("raSelection", true), RBIND("rbind", false), + RCM("rowClassMeet", "rcm", false, false, ReturnType.MULTI_RETURN), REMOVE("remove", false, ReturnType.MULTI_RETURN), + REV("rev", false), ROLL("roll", false), ROUND("round", false), ROW_COUNT_DISTINCT("rowCountDistinct", false), + ROWCUMSUM("rowcumsum", false), ROWINDEXMAX("rowIndexMax", false), ROWINDEXMIN("rowIndexMin", false), + ROWMAX("rowMaxs", false), ROWMEAN("rowMeans", false), ROWMIN("rowMins", false), ROWPROD("rowProds", false), + ROWSD("rowSds", false), ROWSUM("rowSums", false), ROWVAR("rowVars", false), RMSE("rmse", true), + SAMPLE("sample", false), SD("sd", false), SELVARTHRESH("selectByVarThresh", true), SEQ("seq", false), + SES("ses", true), SYMMETRICDIFFERENCE("symmetricDifference", true), SHAPEXPLAINER("shapExplainer", true), + SHERLOCK("sherlock", true), SHERLOCKPREDICT("sherlockPredict", true), SHORTESTPATH("shortestPath", true), + SIGMOID("sigmoid", true), // 1 / (1 + exp(-X)) + SIGN("sign", false), SIN("sin", false), SINH("sinh", false), SLICEFINDER("slicefinder", true), // TODO remove + SLICELINE("sliceLine", true), SLICELINE_DEBUG("sliceLineDebug", true), SLICELINE_EXTRACT("sliceLineExtract", true), + SKEWNESS("skewness", true), SMAPE("smape", true), SMOTE("smote", true), SOFTMAX("softmax", true), + SOLVE("solve", false), SPLIT("split", true), SPLIT_BALANCED("splitBalanced", true), + STABLE_MARRIAGE("stableMarriage", true), STATSNA("statsNA", true), STRATSTATS("stratstats", true), + STEPLM("steplm", true, ReturnType.MULTI_RETURN), STFT("stft", false, ReturnType.MULTI_RETURN), SQRT("sqrt", false), + SQRT_MATRIX("sqrtMatrix", true), SQRT_MATRIX_JAVA("sqrtMatrixJava", false, ReturnType.SINGLE_RETURN), + SUM("sum", false), SVD("svd", false, ReturnType.MULTI_RETURN), TABLE("table", "ctable", false), TAN("tan", false), + TANH("tanh", false), TO_ONE_HOT("toOneHot", true), TOMEKLINK("tomeklink", true), TRACE("trace", "tr", false), + TRANS("t", false), TSNE("tSNE", true), TYPEOF("typeof", false), UNIVAR("univar", true), UNION("union", true), + VAR("var", false), VALUE_SWAP("valueSwap", false), VECTOR_TO_CSV("vectorToCsv", true), WER("wer", true, false), + WINSORIZE("winsorize", true, false), // TODO parameterize w/ prob, min/max val + WINSORIZEAPPLY("winsorizeApply", true, false), // TODO parameterize w/ prob, min/max val + WOE("WoE", true), WOEAPPLY("WoEApply", true), XGBOOST("xgboost", true), XGBOOSTPREDICT("xgboostPredict", true), + XGBOOSTPREDICT_REG("xgboostPredictRegression", true), XGBOOSTPREDICT_CLASS("xgboostPredictClassification", true), XOR("xor", false), // Parameterized functions with parameters diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinPowerTransformTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinPowerTransformTest.java new file mode 100644 index 00000000000..8b020fe8d6b --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinPowerTransformTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.builtin.part2; + +import java.util.HashMap; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.common.Types.ExecType; +import org.apache.sysds.runtime.DMLScriptException; +import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; + +public class BuiltinPowerTransformTest extends AutomatedTestBase { + private static final String TRANSFORM_TEST_NAME = "powerTransform"; + private static final String APPLY_TEST_NAME = "powerTransformApply"; + private static final String TEST_DIR = "functions/builtin/"; + private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinPowerTransformTest.class.getSimpleName() + "/"; + + private static final double REFERENCE_EPS = 1e-4; + private static final double APPLY_EPS = 1e-9; + + @Override + public void setUp() { + addTestConfiguration(TRANSFORM_TEST_NAME, + new TestConfiguration(TEST_CLASS_DIR, TRANSFORM_TEST_NAME, new String[] {"Y", "L", "S"})); + addTestConfiguration(APPLY_TEST_NAME, + new TestConfiguration(TEST_CLASS_DIR, APPLY_TEST_NAME, new String[] {"Y"})); + } + + @Test + public void testPowerTransformYeoJohnsonDefaultDenseCP() { + runPowerTransformYeoJohnsonDefaultDenseTest(ExecType.CP); + } + + @Test + public void testPowerTransformYeoJohnsonDefaultDenseSpark() { + runPowerTransformYeoJohnsonDefaultDenseTest(ExecType.SPARK); + } + + private void runPowerTransformYeoJohnsonDefaultDenseTest(ExecType execType) { + double[][] input = {{-2, 1, 5}, {-1, 1, 5}, {0, 2, 5}, {1, 3, 5}, {2, 6, 5}, {4, 12, 5}}; + runPowerTransformTest(execType, "default", true, input, false); + } + + @Test + public void testPowerTransformBoxCoxUnstandardizedDenseCP() { + double[][] input = {{1.0, 1.0}, {2.0, 1.1}, {3.0, 1.2}, {4.0, 1.3}, {5.0, 1.4}, {6.0, 1.5}, {7.0, 2.0}, + {8.0, 8.0}}; + runPowerTransformTest("box-cox", false, input, false); + } + + @Test + public void testPowerTransformYeoJohnsonLambdaAboveInitialInterval() { + double[][] input = {{0.00}, {0.97}, {0.98}, {0.99}, {1.00}}; + runPowerTransformTest("yeo-johnson", false, input, false); + assertLambdaOutsideInitialInterval(true); + } + + @Test + public void testPowerTransformYeoJohnsonPreservesNaNCP() { + double[][] input = {{-2, 1}, {-1, Double.NaN}, {Double.NaN, 2}, {1, 4}, {2, 8}}; + runPowerTransformTest("yeo-johnson", true, input, false, true, false); + assertNaNPositions(input); + } + + @Test + public void testPowerTransformBoxCoxLambdaBelowInitialInterval() { + double[][] input = {{1.00}, {1.01}, {1.02}, {1.03}, {10.0}}; + runPowerTransformTest("box-cox", false, input, false); + assertLambdaOutsideInitialInterval(false); + } + + @Test + public void testPowerTransformBoxCoxFallsBackToFiniteLambdaCP() { + double[][] input = {{1e-100}, {1e-50}, {1.0}, {1e50}, {1e100}}; + runPowerTransformTest("box-cox", false, input, false, false); + double lambda = readDMLMatrixFromOutputDir("L").get(new CellIndex(1, 1)); + Assert.assertTrue(Double.isFinite(lambda)); + assertNaNPositions(input); + } + + @Test + public void testPowerTransformBoxCoxRejectsNonPositiveInput() { + double[][] input = {{0, 1}, {1, 2}}; + runPowerTransformTest("box-cox", false, input, true); + } + + @Test + public void testPowerTransformApplyYeoJohnsonDenseCP() { + runPowerTransformApplyYeoJohnsonDenseTest(ExecType.CP); + } + + @Test + public void testPowerTransformApplyYeoJohnsonDenseSpark() { + runPowerTransformApplyYeoJohnsonDenseTest(ExecType.SPARK); + } + + private void runPowerTransformApplyYeoJohnsonDenseTest(ExecType execType) { + double[][] input = {{-2, -2, -2}, {-1, -1, -1}, {0, 0, 0}, {1, 1, 1}, {2, 2, 2}}; + double[][] expected = {{-3, -1.5, -1.03944491546724}, {-1.33333333333333, -1, -0.877258872223978}, + {-0.333333333333333, -0.5, -0.6}, {0.128764787039964, 0, 0}, {0.399074859112073, 0.5, 1}}; + runPowerTransformApplyTest(execType, "yeo-johnson", true, input, expected, false); + } + + @Test + public void testPowerTransformApplyBoxCoxDenseCP() { + double[][] input = {{0.5, 0.5, 0.5}, {1.0, 1.0, 1.0}, {2.0, 2.0, 2.0}, {4.0, 4.0, 4.0}, {8.0, 8.0, 8.0}}; + double[][] expected = {{-0.693147180559945, -0.5, -0.375}, {0, 0, 0}, {0.693147180559945, 1, 1.5}, + {1.38629436111989, 3, 7.5}, {2.07944154167984, 7, 31.5}}; + runPowerTransformApplyTest(ExecType.CP, "box-cox", false, input, expected, false); + } + + @Test + public void testPowerTransformApplyBoxCoxPreservesNaNCP() { + double[][] input = {{0.5, Double.NaN, 0.5}, {Double.NaN, 1.0, 1.0}, {2.0, 2.0, Double.NaN}}; + double[][] expected = {{-0.693147180559945, Double.NaN, -0.375}, {Double.NaN, 0, 0}, + {0.693147180559945, 1, Double.NaN}}; + runPowerTransformApplyTest(ExecType.CP, "box-cox", false, input, expected, false); + } + + @Test + public void testPowerTransformApplyBoxCoxRejectsNonPositiveInput() { + double[][] input = {{0, 1, 2}, {1, 2, 3}}; + runPowerTransformApplyTest(ExecType.CP, "box-cox", false, input, null, true); + } + + private void runPowerTransformTest(String method, boolean standardize, double[][] input, boolean shouldFail) { + runPowerTransformTest(ExecType.CP, method, standardize, input, shouldFail, true, true); + } + + private void runPowerTransformTest(ExecType execType, String method, boolean standardize, double[][] input, + boolean shouldFail) { + runPowerTransformTest(execType, method, standardize, input, shouldFail, true, true); + } + + private void runPowerTransformTest(String method, boolean standardize, double[][] input, boolean shouldFail, + boolean compareReference) { + runPowerTransformTest(ExecType.CP, method, standardize, input, shouldFail, compareReference, true); + } + + private void runPowerTransformTest(String method, boolean standardize, double[][] input, boolean shouldFail, + boolean compareReference, boolean compareTransformed) { + runPowerTransformTest(ExecType.CP, method, standardize, input, shouldFail, compareReference, + compareTransformed); + } + + private void runPowerTransformTest(ExecType execType, String method, boolean standardize, double[][] input, + boolean shouldFail, boolean compareReference, boolean compareTransformed) { + ExecMode oldExecMode = setExecMode(execType); + + try { + loadTestConfiguration(getTestConfiguration(TRANSFORM_TEST_NAME)); + + String home = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = home + TRANSFORM_TEST_NAME + ".dml"; + fullRScriptName = home + TRANSFORM_TEST_NAME + ".R"; + programArgs = new String[] {"-args", input("X"), output("Y"), output("L"), output("S"), method, + Boolean.toString(standardize)}; + + if(compareReference) { + String referenceMethod = method.equals("default") ? "yeo-johnson" : method; + rCmd = getRCmd(inputDir(), expectedDir(), referenceMethod, Boolean.toString(standardize)); + } + + writeInputMatrixWithMTD("X", input, true); + runTest(true, shouldFail, shouldFail ? DMLScriptException.class : null, -1); + if(shouldFail) + return; + + if(compareReference) { + runRScript(true); + if(compareTransformed) + compareOutput("Y", REFERENCE_EPS); + compareOutput("L", REFERENCE_EPS); + compareOutput("S", REFERENCE_EPS); + } + } + catch(Exception exception) { + throw new RuntimeException(exception); + } + finally { + resetExecMode(oldExecMode); + } + } + + private void runPowerTransformApplyTest(ExecType execType, String method, boolean standardize, double[][] input, + double[][] expected, boolean shouldFail) { + ExecMode oldExecMode = setExecMode(execType); + + try { + loadTestConfiguration(getTestConfiguration(APPLY_TEST_NAME)); + + String home = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = home + APPLY_TEST_NAME + ".dml"; + programArgs = new String[] {"-args", input("X"), input("L"), input("M"), input("S"), output("Y"), method, + Boolean.toString(standardize)}; + + double[][] L = {{0, 1, 2}}; + double[][] means = {{0.5, 1.0, 1.5}}; + double[][] scales = {{1.5, 2.0, 2.5}}; + + writeInputMatrixWithMTD("X", input, true); + writeInputMatrixWithMTD("L", L, true); + writeInputMatrixWithMTD("M", means, true); + writeInputMatrixWithMTD("S", scales, true); + if(!shouldFail) + writeExpectedMatrix("Y", expected); + + runTest(true, shouldFail, shouldFail ? DMLScriptException.class : null, -1); + if(shouldFail) + return; + + compareResults(APPLY_EPS); + } + catch(Exception exception) { + throw new RuntimeException(exception); + } + finally { + resetExecMode(oldExecMode); + } + } + + private void compareOutput(String name, double tolerance) { + HashMap dmlResult = readDMLMatrixFromOutputDir(name); + HashMap rResult = readRMatrixFromExpectedDir(name); + TestUtils.compareMatrices(dmlResult, rResult, tolerance, "DML", "R"); + } + + private void assertLambdaOutsideInitialInterval(boolean above) { + double lambda = readDMLMatrixFromOutputDir("L").get(new CellIndex(1, 1)); + Assert.assertTrue("Expected lambda outside the initial interval, but was " + lambda, + above ? lambda > 2.0 : lambda < -2.0); + } + + private void assertNaNPositions(double[][] input) { + HashMap output = readDMLMatrixFromOutputDir("Y"); + for(int i = 0; i < input.length; i++) { + for(int j = 0; j < input[i].length; j++) { + double value = output.getOrDefault(new CellIndex(i + 1, j + 1), 0.0); + Assert.assertEquals(Double.isNaN(input[i][j]), Double.isNaN(value)); + } + } + } +} diff --git a/src/test/scripts/functions/builtin/powerTransform.R b/src/test/scripts/functions/builtin/powerTransform.R new file mode 100644 index 00000000000..251fb139911 --- /dev/null +++ b/src/test/scripts/functions/builtin/powerTransform.R @@ -0,0 +1,71 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +library("Matrix") +suppressPackageStartupMessages(library("recipes")) + +args <- commandArgs(TRUE) +X <- as.matrix(readMM(paste(args[1], "X.mtx", sep = ""))) +method <- args[3] +standardize <- as.logical(args[4]) + +colnames(X) <- paste0("V", seq_len(ncol(X))) +data <- as.data.frame(X) +transform <- recipe(~ ., data = data) + +if (method == "box-cox") { + transform <- step_BoxCox( + transform, + all_numeric(), + limits = c(-20, 20), + num_unique = 2 + ) +} else { + transform <- step_YeoJohnson( + transform, + all_numeric(), + limits = c(-20, 20), + num_unique = 2 + ) +} + +fitted <- prep(transform, training = data) +estimates <- tidy(fitted, number = 1) +lambdas <- matrix(1.0, nrow = 1, ncol = ncol(X)) +lambdas[1, match(estimates$terms, colnames(X))] <- estimates$value +Y <- as.matrix(bake(fitted, new_data = data)) + +if (standardize) { + observed <- colSums(!is.na(Y)) + means <- colMeans(Y, na.rm = TRUE) + means[is.nan(means)] <- 0 + centered <- sweep(Y, 2, means) + scales <- sqrt(colSums(centered^2, na.rm = TRUE) / observed) + scales[observed == 0 | scales == 0 | is.nan(scales)] <- 1 + Y <- sweep(centered, 2, scales, "/") + state <- rbind(means, scales) +} else { + state <- matrix(0.0, nrow = 2, ncol = ncol(X)) +} + +writeMM(as(Y, "CsparseMatrix"), paste(args[2], "Y", sep = "")) +writeMM(as(lambdas, "CsparseMatrix"), paste(args[2], "L", sep = "")) +writeMM(as(state, "CsparseMatrix"), paste(args[2], "S", sep = "")) diff --git a/src/test/scripts/functions/builtin/powerTransform.dml b/src/test/scripts/functions/builtin/powerTransform.dml new file mode 100644 index 00000000000..d7dfba4759d --- /dev/null +++ b/src/test/scripts/functions/builtin/powerTransform.dml @@ -0,0 +1,48 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1); + +if ($5 == "default") { + [Y, lambdas, means, scales] = powerTransform(X=X); + standardize = TRUE; +} +else { + standardize = as.boolean($6); + [Y, lambdas, means, scales] = powerTransform( + X=X, + method=$5, + standardize=standardize + ); +} + +if (standardize) { + state = rbind(means, scales); +} +else { + assert(nrow(means) == 0 & ncol(means) == 0); + assert(nrow(scales) == 0 & ncol(scales) == 0); + state = matrix(0.0, rows=2, cols=ncol(X)); +} + +write(Y, $2); +write(lambdas, $3); +write(state, $4); diff --git a/src/test/scripts/functions/builtin/powerTransformApply.dml b/src/test/scripts/functions/builtin/powerTransformApply.dml new file mode 100644 index 00000000000..87615b8daef --- /dev/null +++ b/src/test/scripts/functions/builtin/powerTransformApply.dml @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1); +lambdas = read($2); +means = read($3); +scales = read($4); + +if (!as.boolean($7)) { + means = matrix(0.0, rows=0, cols=0); + scales = matrix(0.0, rows=0, cols=0); +} + +Y = powerTransformApply(X, lambdas, means, scales, $6); +write(Y, $5); diff --git a/src/test/scripts/installDependencies.R b/src/test/scripts/installDependencies.R index 60642fa8ed4..0bbe8960682 100644 --- a/src/test/scripts/installDependencies.R +++ b/src/test/scripts/installDependencies.R @@ -52,6 +52,7 @@ custom_install("boot"); custom_install("matrixStats"); custom_install("outliers"); custom_install("caret"); +custom_install("recipes"); custom_install("sigmoid"); custom_install("DescTools"); custom_install("mice");