[SYSTEMDS-3857] set get names on dataframes

docs/site/dml-language-reference.md

@@ -2068,10 +2068,12 @@ The following example uses <code>transformapply()</code> with the input matrix a
 
 **Table F5**: Frame processing built-in functions
 
-Function | Description | Parameters | Example
--------- | ----------- | ---------- | -------
-map() | It will execute the given lambda expression on a frame (cell, row or column wise). | Input: (X &lt;frame&gt;, y &lt;String&gt;, \[margin &lt;int&gt;\]) <br/>Output: &lt;frame&gt;. <br/> X is a frame and <br/>y is a String containing the lambda expression to be executed on frame X. <br/> margin - how to apply the lambda expression (0 indicates each cell, 1 - rows, 2 - columns). Output matrix dimensions are always equal to the input. | [map](#map) 
-tokenize() | Transforms a frame to tokenized frame using specification. Tokenization is valid only for string columns. | Input:<br/> target = &lt;frame&gt; <br/> spec = &lt;json specification&gt; <br/> Outputs: &lt;matrix&gt;, &lt;frame&gt; | [tokenize](#tokenize)
+Function | Description                                                                                               | Parameters                                                                                                                                                                                                                                                                                                                                          | Example
+-------- |-----------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------
+map() | It will execute the given lambda expression on a frame (cell, row or column wise).                        | Input: (X &lt;frame&gt;, y &lt;String&gt;, \[margin &lt;int&gt;\]) <br/>Output: &lt;frame&gt;. <br/> X is a frame and <br/>y is a String containing the lambda expression to be executed on frame X. <br/> margin - how to apply the lambda expression (0 indicates each cell, 1 - rows, 2 - columns). Output matrix dimensions are always equal to the input. | [map](#map) 
+tokenize() | Transforms a frame to tokenized frame using specification. Tokenization is valid only for string columns. | Input:<br/> target = &lt;frame&gt; <br/> spec = &lt;json specification&gt; <br/> Outputs: &lt;matrix&gt;, &lt;frame&gt;                                                                                                                                                                                                                             | [tokenize](#tokenize)
+getNames() | Returns the column names of a frame as a single-row frame.                                                | Input: X &lt;frame&gt; <br/> Output: &lt;frame&gt;                                                                                                                                                                                                                                                                                                  | N = getNames(X)
+setNames() | Sets the column names of a frame from a single-row frame containing string values. | Input:<br/> X = &lt;frame&gt;<br/> N = &lt;frame&gt;<br/> Output:&lt;frame&gt;                                                                                                                                                                                                                                                                      | Y = setNames(X, N)
 
 #### map
 

src/main/java/org/apache/sysds/api/jmlc/PreparedScript.java

@@ -397,6 +397,8 @@ public void setFrame(String varname, FrameBlock frame, boolean reuse) {
 		MetaDataFormat meta = new MetaDataFormat(mc, FileFormat.BINARY);
 		FrameObject fo = new FrameObject(OptimizerUtils.getUniqueTempFileName(), meta);
 		fo.acquireModify(frame);
+		fo.setSchema(frame.getSchema());
+		fo.setColumnNames(frame.getColumnNames());
 		fo.release();
 
 		//put create matrix wrapper into symbol table

src/main/java/org/apache/sysds/api/mlcontext/MLContextConversionUtil.java

@@ -189,6 +189,7 @@ public static FrameObject frameBlockToFrameObject(String variableName, FrameBloc
 			FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), mtd,
 				frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
 			frameObject.acquireModify(frameBlock);
+			frameObject.setColumnNames(frameBlock.getColumnNames());
 			frameObject.release();
 			return frameObject;
 		} catch (DMLRuntimeException e) {
@@ -299,6 +300,7 @@ public static FrameObject binaryBlocksToFrameObject(JavaPairRDD<Long, FrameBlock
 
 		FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
 			new MetaDataFormat(mc, FileFormat.BINARY), schema);
+
 		frameObject.setRDDHandle(new RDDObject(binaryBlocks));
 		return frameObject;
 	}

src/main/java/org/apache/sysds/common/Builtins.java

@@ -154,6 +154,7 @@ public enum Builtins {
 	GARCH("garch", true),
 	GAUSSIAN_CLASSIFIER("gaussianClassifier", true),
 	GET_ACCURACY("getAccuracy", true),
+	GET_NAMES("getNames", false),
 	GET_CATEGORICAL_MASK("getCategoricalMask", false),
 	GLM("glm", true),
 	GLM_PREDICT("glmPredict", true),
@@ -310,6 +311,7 @@ public enum Builtins {
 	SELVARTHRESH("selectByVarThresh", true),
 	SEQ("seq", false),
 	SES("ses", true),
+	SET_NAMES("setNames", false),
 	SYMMETRICDIFFERENCE("symmetricDifference", true),
 	SHAPEXPLAINER("shapExplainer", true),
 	SHERLOCK("sherlock", true),

src/main/java/org/apache/sysds/common/Opcodes.java

@@ -352,6 +352,7 @@ public enum Opcodes {
 	MAPPM("map+*", InstructionType.Binary),
 	MAPMINUSMULT("map-*", InstructionType.Binary),
 	MAPDROPINVALIDLENGTH("mapdropInvalidLength", InstructionType.Binary),
+	SET_COLNAMES("set_colnames", InstructionType.Binary),
 
 	MAPGT("map>", InstructionType.Binary),
 	MAPGE("map>=", InstructionType.Binary),

src/main/java/org/apache/sysds/common/Types.java

@@ -641,7 +641,8 @@ public enum OpOp2 {
 		MINUS1_MULT(false), //1-X*Y
 		GET_CATEGORICAL_MASK(false), // get transformation mask
 		QUANTIZE_COMPRESS(false), //quantization-fused compression
-		UNION_DISTINCT(false);
+		UNION_DISTINCT(false),
+		SET_COLNAMES(false);
 
 		private final boolean _validOuter;
 

src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java

@@ -1095,13 +1095,34 @@ else if( getAllExpr().length == 2 ) { //binary
 		case TYPEOF:
 		case DETECTSCHEMA:
 		case COLNAMES:
+		case GET_NAMES:
 			checkNumParameters(1);
 			checkMatrixFrameParam(getFirstExpr());
 			output.setDataType(DataType.FRAME);
 			output.setDimensions(1, id.getDim2());
 			output.setBlocksize (id.getBlocksize());
 			output.setValueType(ValueType.STRING);
 			break;
+		case SET_NAMES:
+			//check if we use 2 parameters (Frame on which nemas are set and vector for names)
+			checkNumParameters(2);
+
+			// check if first paramters is a frame
+			checkMatrixFrameParam(getFirstExpr());
+
+			// check if second paramters is a vector 1xn Frame
+			checkMatrixFrameParam(getSecondExpr());
+
+			//output should be a frame
+			output.setDataType(DataType.FRAME);
+
+
+			checkMatrixFrameParam(getFirstExpr());
+			output.setDataType(DataType.FRAME);
+			output.setDimensions(id.getDim1(), id.getDim2());
+			output.setBlocksize (id.getBlocksize());
+			output.setDataType(DataType.FRAME);
+			break;
 		case CAST_AS_FRAME:
 			// operation as.frame
 			// overloaded to take either one argument or 2 where second is column names

src/main/java/org/apache/sysds/parser/DMLTranslator.java

@@ -2763,6 +2763,34 @@ else if ( in.length == 2 )
 		case TYPEOF:
 		case DET:
 		case DETECTSCHEMA:
+			currBuiltinOp = new UnaryOp(
+					target.getName(),
+					target.getDataType(),
+					target.getValueType(),
+					OpOp1.valueOf(source.getOpCode().name()),
+					expr
+			);
+			break;
+
+		case SET_NAMES:
+			currBuiltinOp = new BinaryOp(
+					target.getName(),
+					target.getDataType(),
+					target.getValueType(),
+					OpOp2.SET_COLNAMES,
+					expr,
+					expr2
+			);
+			break;
+
+		case GET_NAMES:
+			currBuiltinOp = new UnaryOp(
+					target.getName(),
+					target.getDataType(),
+					target.getValueType(),
+					OpOp1.COLNAMES, expr
+			);
+			break;
 		case COLNAMES:
 			currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(),
 				target.getValueType(), OpOp1.valueOf(source.getOpCode().name()), expr);

src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java

@@ -38,10 +38,7 @@
 import org.apache.sysds.runtime.instructions.ooc.OOCStream;
 import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue;
 import org.apache.sysds.runtime.instructions.spark.data.RDDObject;
-import org.apache.sysds.runtime.io.FileFormatProperties;
-import org.apache.sysds.runtime.io.FrameReaderFactory;
-import org.apache.sysds.runtime.io.FrameWriter;
-import org.apache.sysds.runtime.io.FrameWriterFactory;
+import org.apache.sysds.runtime.io.*;
 import org.apache.sysds.runtime.lineage.LineageItem;
 import org.apache.sysds.runtime.lineage.LineageRecomputeUtils;
 import org.apache.sysds.runtime.meta.DataCharacteristics;
@@ -61,7 +58,9 @@
 	private static final long serialVersionUID = 1755082174281927785L;
 
 	private ValueType[] _schema = null;
-
+
+	private String[] _colnames = null;
+
 	protected FrameObject() {
 		super(DataType.FRAME, ValueType.STRING);
 	}
@@ -85,18 +84,19 @@
 	}
 
 	/**
-	 * Copy constructor that copies meta data but NO data.
-	 * 
+	 * Copy constructor that copies meta data and column names but NO data.
+	 *
 	 * @param fo frame object
 	 */
 	public FrameObject(FrameObject fo) {
 		super(fo);
 
 		MetaDataFormat metaOld = (MetaDataFormat) fo.getMetaData();
 		_metaData = new MetaDataFormat(
-			new MatrixCharacteristics(metaOld.getDataCharacteristics()),
-			metaOld.getFileFormat());
-		_schema = fo._schema.clone();
+				new MatrixCharacteristics(metaOld.getDataCharacteristics()),
+				metaOld.getFileFormat());
+		_schema = fo._schema != null ? fo._schema.clone() : null;
+		_colnames = fo._colnames != null ? fo._colnames.clone() : null;
 	}
 
 	@Override
@@ -112,8 +112,8 @@
 	 * @return schema of value types
 	 */
 	public ValueType[] getSchema(int cl, int cu) {
-		return (_schema!=null && _schema.length>cu) ? Arrays.copyOfRange(_schema, cl, cu+1) :
-			UtilFunctions.nCopies(cu-cl+1, ValueType.STRING);
+		return (_schema != null && _schema.length > cu) ? Arrays.copyOfRange(_schema, cl, cu + 1) :
+				UtilFunctions.nCopies(cu - cl + 1, ValueType.STRING);
 	}
 
 	/**
@@ -125,10 +125,14 @@
 	 */
 	public ValueType[] mergeSchemas(FrameObject fo) {
 		return ArrayUtils.addAll(
-			(_schema!=null) ? _schema : UtilFunctions.nCopies((int)getNumColumns(), ValueType.STRING), 
-			(fo._schema!=null) ? fo._schema : UtilFunctions.nCopies((int)fo.getNumColumns(), ValueType.STRING));
-	} 
-
+				(_schema != null) ? _schema : UtilFunctions.nCopies((int) getNumColumns(), ValueType.STRING),
+				(fo._schema != null) ? fo._schema : UtilFunctions.nCopies((int) fo.getNumColumns(), ValueType.STRING));
+	}
+
+	/**
+	 *
+	 * @param schema
+	 */
 	public void setSchema(String schema) {
 		if( schema.equals("*") ) {
 			//populate default schema
@@ -154,17 +158,44 @@
 	public void setSchema(ValueType[] schema) {
 		_schema = schema;
 	}
-
+
+	public String[] getColumnNames() {
+		return _colnames;
+	}
+
+	/**
+	 * Obtain column names
+	 *
+	 * @param cl column lower bound, inclusive
+	 * @param cu column upper bound, inclusive
+	 * @return column names
+	 */
+	public String[] getColumnNames(int cl, int cu) {
+		return (_colnames != null && _colnames.length > cu)
+				? Arrays.copyOfRange(_colnames, cl, cu + 1)
+				: FrameBlock.createColNames(cu - cl + 1);
+	}
+
+	/**
+	 *
+	 * @param colNames
+	 */
+	public void setColumnNames(String[] colNames) {
+		_colnames = colNames != null
+				? colNames.clone()
+				: null;
+	}
+
 	@Override
 	public void refreshMetaData() {
-		if ( _data == null || _metaData ==null ) //refresh only for existing data
-			throw new DMLRuntimeException("Cannot refresh meta data because there is no data or meta data. "); 
+		if (_data == null || _metaData == null) //refresh only for existing data
+			throw new DMLRuntimeException("Cannot refresh meta data because there is no data or meta data. ");
 
 		//update matrix characteristics
 		DataCharacteristics dc = _metaData.getDataCharacteristics();
-		dc.setDimension( _data.getNumRows(),_data.getNumColumns() );
-		dc.setNonZeros(_data.getNumRows()*_data.getNumColumns());
-		
+		dc.setDimension(_data.getNumRows(), _data.getNumColumns());
+		dc.setNonZeros(_data.getNumRows() * _data.getNumColumns());
+
 		//update schema information
 		_schema = _data.getSchema();
 	}
@@ -208,15 +239,29 @@
 		if(data == null)
 			throw new IOException("Unable to load frame from file: " + fname);
 
+		FileFormat format = iimd.getFileFormat();
+
 		//Delta and CSV discover dimensions (and Delta also schema) at read time, so
 		//refresh the cached metadata to reflect the materialized frame block.
-		if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) {
+		if(format == FileFormat.CSV || format == FileFormat.DELTA) {
 			_metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(data.getDataCharacteristics(),
-				iimd.getFileFormat()) : new MetaData(data.getDataCharacteristics());
-			if(iimd.getFileFormat() == FileFormat.DELTA)
+					format) : new MetaData(data.getDataCharacteristics());
+
+			if(format == FileFormat.DELTA)
 				_schema = data.getSchema();
 		}
 
+		if (format == FileFormat.PARQUET)
+			_schema = data.getSchema();
+
+		if (_colnames == null && (format == FileFormat.CSV || format == FileFormat.PARQUET)) {
+			String[] columnNames = data.getColumnNames();
+
+			if (columnNames != null) {
+				setColumnNames(columnNames);
+			}
+		}
+
 		return data;
 	}
 

src/main/java/org/apache/sysds/runtime/controlprogram/context/ExecutionContext.java

@@ -703,6 +703,7 @@ public static FrameObject createFrameObject(FrameBlock fb) {
 		ret.acquireModify(fb);
 		ret.setMetaData(new MetaDataFormat(new MatrixCharacteristics(
 			fb.getNumRows(), fb.getNumColumns()), FileFormat.BINARY));
+		ret.setColumnNames(fb.getColumnNames());
 		ret.release();
 		return ret;
 	}