Package org.apache.sysds.runtime.util
Class UtilFunctions
- java.lang.Object
- 
- org.apache.sysds.runtime.util.UtilFunctions
 
- 
 public class UtilFunctions extends Object 
- 
- 
Field SummaryFields Modifier and Type Field Description static longADD_PRIME1static intDIVIDE_PRIMEstatic doubleDOUBLE_EPS
 - 
Constructor SummaryConstructors Constructor Description UtilFunctions()
 - 
Method SummaryAll Methods Static Methods Concrete Methods Modifier and Type Method Description static StringaddTimeToDate(String dateString, int amountToAdd, String timeformat)static StringcolumnStringToCSVString(String input, String separator)static intcompareTo(Types.ValueType vt, Object in1, Object in2)static intcompareVersion(String version1, String version2)Compares two version strings of format x.y.z, where x is major, y is minor, and z is maintenance release.static longcomputeBlockIndex(long cellIndex, int blockSize)Computes the 1-based block index based on the global cell index and block size meta data.static longcomputeBlockNumber(int[] ix, long[] dims, int blen)Calculates the number of the block this index refers to (basically a linearisation).static intcomputeBlockSize(long len, long blockIndex, long blockSize)Computes the actual block size based on matrix dimension, block index, and block size meta data.static intcomputeCellInBlock(long cellIndex, int blockSize)Computes the 0-based cell-in-block index based on the global cell index and block size meta data.static longcomputeCellIndex(long blockIndex, int blockSize, int cellInBlock)Computes the global 1-based cell index based on the block index, block size meta data, and specific 0-based in-block cell index.static long[]computeNextTensorIndexes(TensorCharacteristics tc, long[] ix)Computes the next tensor indexes array.static intcomputeNnz(double[] a, int ai, int len)static intcomputeNnz(float[] a, int ai, int len)static intcomputeNnz(int[] a, int ai, int len)static intcomputeNnz(long[] a, int ai, int len)static intcomputeNnz(String[] a, int ai, int len)static intcomputeNnz(BitSet a, int ai, int len)static longcomputeNnz(SparseBlock a, int[] aix, int ai, int alen)static voidcomputeSliceInfo(TensorCharacteristics tc, long[] blockIx, int[] outDims, int[] offset)Computes the slice dimensions and offsets for the block slice of another tensor with the size given byTensorCharacteristics.static long[]computeTensorIndexes(TensorCharacteristics tc, long blockIndex)Computes the tensor indexes array given a blockIndex we ant to compute.static booleancontainsZero(double[] data, int pos, int len)static double[]convertStringToDoubleArray(String[] original)static String[]copyAsStringToArray(String[] input, Object value)static Types.ValueType[]copyOf(Types.ValueType[] schema1, Types.ValueType[] schema2)static intcountNonZeros(double[] data, int pos, int len)static StringdateFormat(long date, String outputFormat)static StringdateFormat(String dateString, String outputFormat)static StringdateFormat(String dateString, String inputFormat, String outputFormat)static ObjectdoubleToObject(Types.ValueType vt, double in)static ObjectdoubleToObject(Types.ValueType vt, double in, boolean sparse)static StringformatMemorySize(long arg)Format a memory size with g/m/k quantifiers into its number representation.static intfrequency(Types.ValueType[] schema, Types.ValueType vt)static FrameBlockgenerateRandomFrameBlock(int rows, int cols, Types.ValueType[] schema, Random random)Generates a random FrameBlock with given parameters.static ObjectgenerateRandomValueFromValueType(Types.ValueType valueType, Random random)Generates a random value for a given Value Typestatic ArrayList<Integer>getAlignedBlockSizes(int len, int k, int align)static intgetAsciiAtIdx(String s, int idx)static ArrayList<Integer>getBalancedBlockSizesDefault(int len, int k, boolean constK)static voidgetBlockBounds(TensorIndexes ix, long[] dims, int blen, int[] lower, int[] upper)static int[]getBlockSizes(int num, int numBlocks)static String[]getDominantDateFormat(String[] values)static doublegetDouble(Object obj)static intgetEndIndex(int arrayLength, int startIndex, int blockSize)static IndexRangegetSelectedRangeForZeroOut(IndexedMatrixValue in, int blen, IndexRange indexRange)static IndexRangegetSelectedRangeForZeroOut(Pair<Long,FrameBlock> in, int blen, IndexRange indexRange, long lSrcRowIndex, long lDestRowIndex)static int[]getSeqArray(int low, int up, int incr)Obtain sequence arraystatic longgetSeqLength(double from, double to, double incr)static longgetSeqLength(double from, double to, double incr, boolean check)static List<Integer>getSeqList(int low, int up, int incr)Obtain sequence liststatic int[]getSortedSampleIndexes(int range, int sampleSize)static int[]getSortedSampleIndexes(int range, int sampleSize, long seed)static String[]getSplittedStringAsArray(String input)static String[]getTimestamp(String[] values)static intintHashCode(int key1, int key2)static intintHashCodeRobust(int key1, int key2)static booleanisBoolean(String str)static StringisDateColumn(String values)static booleanisInBlockRange(MatrixIndexes ix, int blen, long rl, long ru, long cl, long cu)static booleanisInBlockRange(MatrixIndexes ix, int blen, IndexRange ixrange)static booleanisInFrameBlockRange(Long ix, int blen, long rl, long ru)static booleanisInFrameBlockRange(Long ix, int blen, IndexRange ixrange)static booleanisIntegerNumber(String str)static booleanisNonZero(Object obj)static booleanisSpecial(double value)static doublejaccardSim(String x, String y)static intlongHashCode(long key1)static intlongHashCode(long key1, long key2)Returns the hash code for a long-long pair.static intlongHashCode(long key1, long key2, long key3)Returns the hash code for a long-long-long triple.static bytemax(byte[] array)static Types.ValueType[]nCopies(int n, Types.ValueType vt)static intnextIntPow2(int in)static doubleobjectToDouble(Types.ValueType vt, Object in)static doubleobjectToDoubleSafe(Types.ValueType vt, Object in)static ObjectobjectToObject(Types.ValueType vt, Object in)static ObjectobjectToObject(Types.ValueType vt, Object in, boolean ignoreNull)static StringobjectToString(Object in)static StringobjectToString(Object in, boolean ignoreNull)Convert object to stringstatic longparseMemorySize(String arg)Parses a memory size with optional g/m/k quantifiers into its number representation.static doubleparseToDouble(String str, Set<String> isNan)Safe double parsing including handling of NAs.static intparseToInt(String str)static longparseToLong(String str)static longpow(int base, int exp)static longprod(int[] arr)static longprod(int[] arr, int off)static longprod(long[] arr)static Stringquote(String s)static introundToNext(int val, int factor)static String[]splitRecodeEntry(String s)static ObjectstringToObject(Types.ValueType vt, String in)static Types.ValueType[]stringToValueType(String[] schemaValues)Generates a ValueType array from a String arraystatic inttoInt(double val)static inttoInt(Object obj)static longtoLong(double val)static longtoMillis(String dateString)static longtoMillis(String dateString, String dateFormat)static String[]toStringArray(Object[] original)static Stringunquote(String s)
 
- 
- 
- 
Field Detail- 
DOUBLE_EPSpublic static final double DOUBLE_EPS 
 - 
ADD_PRIME1public static final long ADD_PRIME1 - See Also:
- Constant Field Values
 
 - 
DIVIDE_PRIMEpublic static final int DIVIDE_PRIME - See Also:
- Constant Field Values
 
 
- 
 - 
Method Detail- 
intHashCodepublic static int intHashCode(int key1, int key2)
 - 
intHashCodeRobustpublic static int intHashCodeRobust(int key1, int key2)
 - 
longHashCodepublic static int longHashCode(long key1) 
 - 
longHashCodepublic static int longHashCode(long key1, long key2)Returns the hash code for a long-long pair. This is the default hash function for the keys of a distributed matrix in MR/Spark.- Parameters:
- key1- first long key
- key2- second long key
- Returns:
- hash code
 
 - 
longHashCodepublic static int longHashCode(long key1, long key2, long key3)Returns the hash code for a long-long-long triple. This is the default hash function for the keys of a distributed matrix in MR/Spark.- Parameters:
- key1- first long key
- key2- second long key
- key3- third long key
- Returns:
- hash code
 
 - 
nextIntPow2public static int nextIntPow2(int in) 
 - 
powpublic static long pow(int base, int exp)
 - 
computeBlockIndexpublic static long computeBlockIndex(long cellIndex, int blockSize)Computes the 1-based block index based on the global cell index and block size meta data. See computeCellIndex for the inverse operation.- Parameters:
- cellIndex- global cell index
- blockSize- block size
- Returns:
- 1-based block index
 
 - 
computeCellInBlockpublic static int computeCellInBlock(long cellIndex, int blockSize)Computes the 0-based cell-in-block index based on the global cell index and block size meta data. See computeCellIndex for the inverse operation.- Parameters:
- cellIndex- global cell index
- blockSize- block size
- Returns:
- 0-based cell-in-block index
 
 - 
computeCellIndexpublic static long computeCellIndex(long blockIndex, int blockSize, int cellInBlock)Computes the global 1-based cell index based on the block index, block size meta data, and specific 0-based in-block cell index. NOTE: this is equivalent to cellIndexCalculation.- Parameters:
- blockIndex- block index
- blockSize- block size
- cellInBlock- 0-based cell-in-block index
- Returns:
- global 1-based cell index
 
 - 
computeBlockSizepublic static int computeBlockSize(long len, long blockIndex, long blockSize)Computes the actual block size based on matrix dimension, block index, and block size meta data. For boundary blocks, the actual block size is less or equal than the block size meta data; otherwise they are identical.- Parameters:
- len- matrix dimension
- blockIndex- block index
- blockSize- block size metadata
- Returns:
- actual block size
 
 - 
computeNextTensorIndexespublic static long[] computeNextTensorIndexes(TensorCharacteristics tc, long[] ix) Computes the next tensor indexes array.- Parameters:
- tc- the tensor characteristics
- ix- the tensor indexes array (will be changed)
- Returns:
- the tensor indexes array (changed)
 
 - 
computeTensorIndexespublic static long[] computeTensorIndexes(TensorCharacteristics tc, long blockIndex) Computes the tensor indexes array given a blockIndex we ant to compute. Note that if a sequence of tensor indexes array will be computed, it is faster to useUtilFunctions.computeNextTensorIndexes(TensorCharacteristics,long[]).- Parameters:
- tc- the tensor characteristics
- blockIndex- the number of the block ([0-- tc.getNumBlocks()[ valid)
- Returns:
- the tensor index array
 
 - 
computeSliceInfopublic static void computeSliceInfo(TensorCharacteristics tc, long[] blockIx, int[] outDims, int[] offset) Computes the slice dimensions and offsets for the block slice of another tensor with the size given byTensorCharacteristics.- Parameters:
- tc- tensor characteristics of the block to slice
- blockIx- the tensor block index
- outDims- the slice dimension size
- offset- the offset where the slice should start
 
 - 
computeBlockNumberpublic static long computeBlockNumber(int[] ix, long[] dims, int blen)Calculates the number of the block this index refers to (basically a linearisation).- Parameters:
- ix- the dimensional indexes
- dims- length of dimensions
- blen- length of blocks
- Returns:
- the number of the block
 
 - 
getBalancedBlockSizesDefaultpublic static ArrayList<Integer> getBalancedBlockSizesDefault(int len, int k, boolean constK) 
 - 
getAlignedBlockSizespublic static ArrayList<Integer> getAlignedBlockSizes(int len, int k, int align) 
 - 
isInBlockRangepublic static boolean isInBlockRange(MatrixIndexes ix, int blen, long rl, long ru, long cl, long cu) 
 - 
isInFrameBlockRangepublic static boolean isInFrameBlockRange(Long ix, int blen, long rl, long ru) 
 - 
isInBlockRangepublic static boolean isInBlockRange(MatrixIndexes ix, int blen, IndexRange ixrange) 
 - 
isInFrameBlockRangepublic static boolean isInFrameBlockRange(Long ix, int blen, IndexRange ixrange) 
 - 
getSelectedRangeForZeroOutpublic static IndexRange getSelectedRangeForZeroOut(IndexedMatrixValue in, int blen, IndexRange indexRange) 
 - 
getSelectedRangeForZeroOutpublic static IndexRange getSelectedRangeForZeroOut(Pair<Long,FrameBlock> in, int blen, IndexRange indexRange, long lSrcRowIndex, long lDestRowIndex) 
 - 
parseToDoublepublic static double parseToDouble(String str, Set<String> isNan) Safe double parsing including handling of NAs. Previously, we also used this wrapper for handling thread contention in multi-threaded environments because Double.parseDouble relied on a synchronized cache (which was replaced with thread-local caches in JDK8).- Parameters:
- str- string to parse to double
- isNan- collection of Nan string which if encountered should be parsed to nan value
- Returns:
- double value
 
 - 
parseToIntpublic static int parseToInt(String str) 
 - 
parseToLongpublic static long parseToLong(String str) 
 - 
toIntpublic static int toInt(double val) 
 - 
toLongpublic static long toLong(double val) 
 - 
toIntpublic static int toInt(Object obj) 
 - 
getSeqLengthpublic static long getSeqLength(double from, double to, double incr)
 - 
getSeqLengthpublic static long getSeqLength(double from, double to, double incr, boolean check)
 - 
getSeqListpublic static List<Integer> getSeqList(int low, int up, int incr) Obtain sequence list- Parameters:
- low- lower bound (inclusive)
- up- upper bound (inclusive)
- incr- increment
- Returns:
- list of integers
 
 - 
getSeqArraypublic static int[] getSeqArray(int low, int up, int incr)Obtain sequence array- Parameters:
- low- lower bound (inclusive)
- up- upper bound (inclusive)
- incr- increment
- Returns:
- array of integers
 
 - 
roundToNextpublic static int roundToNext(int val, int factor)
 - 
doubleToObjectpublic static Object doubleToObject(Types.ValueType vt, double in) 
 - 
doubleToObjectpublic static Object doubleToObject(Types.ValueType vt, double in, boolean sparse) 
 - 
stringToObjectpublic static Object stringToObject(Types.ValueType vt, String in) 
 - 
objectToDoubleSafepublic static double objectToDoubleSafe(Types.ValueType vt, Object in) 
 - 
objectToDoublepublic static double objectToDouble(Types.ValueType vt, Object in) 
 - 
objectToStringpublic static String objectToString(Object in, boolean ignoreNull) Convert object to string- Parameters:
- in- object
- ignoreNull- If this flag has set, it will ignore null. This flag is mainly used in merge functionality to override data with "null" data.
- Returns:
- string representation of object
 
 - 
objectToObjectpublic static Object objectToObject(Types.ValueType vt, Object in) 
 - 
objectToObjectpublic static Object objectToObject(Types.ValueType vt, Object in, boolean ignoreNull) 
 - 
compareTopublic static int compareTo(Types.ValueType vt, Object in1, Object in2) 
 - 
compareVersionpublic static int compareVersion(String version1, String version2) Compares two version strings of format x.y.z, where x is major, y is minor, and z is maintenance release.- Parameters:
- version1- first version string
- version2- second version string
- Returns:
- 1 if version1 greater, -1 if version2 greater, 0 if equal
 
 - 
isBooleanpublic static boolean isBoolean(String str) 
 - 
isIntegerNumberpublic static boolean isIntegerNumber(String str) 
 - 
isSpecialpublic static boolean isSpecial(double value) 
 - 
getSortedSampleIndexespublic static int[] getSortedSampleIndexes(int range, int sampleSize)
 - 
getSortedSampleIndexespublic static int[] getSortedSampleIndexes(int range, int sampleSize, long seed)
 - 
maxpublic static byte max(byte[] array) 
 - 
getAsciiAtIdxpublic static int getAsciiAtIdx(String s, int idx) 
 - 
parseMemorySizepublic static long parseMemorySize(String arg) Parses a memory size with optional g/m/k quantifiers into its number representation.- Parameters:
- arg- memory size as readable string
- Returns:
- byte count of memory size
 
 - 
formatMemorySizepublic static String formatMemorySize(long arg) Format a memory size with g/m/k quantifiers into its number representation.- Parameters:
- arg- byte count of memory size
- Returns:
- memory size as readable string
 
 - 
getDoublepublic static double getDouble(Object obj) 
 - 
isNonZeropublic static boolean isNonZero(Object obj) 
 - 
computeNnzpublic static int computeNnz(double[] a, int ai, int len)
 - 
computeNnzpublic static int computeNnz(float[] a, int ai, int len)
 - 
computeNnzpublic static int computeNnz(long[] a, int ai, int len)
 - 
computeNnzpublic static int computeNnz(int[] a, int ai, int len)
 - 
computeNnzpublic static int computeNnz(BitSet a, int ai, int len) 
 - 
computeNnzpublic static int computeNnz(String[] a, int ai, int len) 
 - 
computeNnzpublic static long computeNnz(SparseBlock a, int[] aix, int ai, int alen) 
 - 
nCopiespublic static Types.ValueType[] nCopies(int n, Types.ValueType vt) 
 - 
frequencypublic static int frequency(Types.ValueType[] schema, Types.ValueType vt) 
 - 
copyOfpublic static Types.ValueType[] copyOf(Types.ValueType[] schema1, Types.ValueType[] schema2) 
 - 
countNonZerospublic static int countNonZeros(double[] data, int pos, int len)
 - 
containsZeropublic static boolean containsZero(double[] data, int pos, int len)
 - 
prodpublic static long prod(long[] arr) 
 - 
prodpublic static long prod(int[] arr) 
 - 
prodpublic static long prod(int[] arr, int off)
 - 
getBlockBoundspublic static void getBlockBounds(TensorIndexes ix, long[] dims, int blen, int[] lower, int[] upper) 
 - 
toMillispublic static long toMillis(String dateString) 
 - 
dateFormatpublic static String dateFormat(String dateString, String inputFormat, String outputFormat) 
 - 
addTimeToDatepublic static String addTimeToDate(String dateString, int amountToAdd, String timeformat) 
 - 
columnStringToCSVStringpublic static String columnStringToCSVString(String input, String separator) 
 - 
generateRandomFrameBlockpublic static FrameBlock generateRandomFrameBlock(int rows, int cols, Types.ValueType[] schema, Random random) Generates a random FrameBlock with given parameters.- Parameters:
- rows- frame rows
- cols- frame cols
- schema- frame schema
- random- random number generator
- Returns:
- FrameBlock
 
 - 
generateRandomValueFromValueTypepublic static Object generateRandomValueFromValueType(Types.ValueType valueType, Random random) Generates a random value for a given Value Type- Parameters:
- valueType- the ValueType of which to generate the value
- random- random number generator
- Returns:
- Object
 
 - 
stringToValueTypepublic static Types.ValueType[] stringToValueType(String[] schemaValues) Generates a ValueType array from a String array- Parameters:
- schemaValues- the string schema of which to generate the ValueType
- Returns:
- ValueType[]
 
 - 
getEndIndexpublic static int getEndIndex(int arrayLength, int startIndex, int blockSize)
 - 
getBlockSizespublic static int[] getBlockSizes(int num, int numBlocks)
 - 
convertStringToDoubleArraypublic static double[] convertStringToDoubleArray(String[] original) 
 
- 
 
-