Package org.apache.sysds.runtime.util
Class UtilFunctions
- java.lang.Object
-
- org.apache.sysds.runtime.util.UtilFunctions
-
public class UtilFunctions extends Object
-
-
Field Summary
Fields Modifier and Type Field Description static longADD_PRIME1static intDIVIDE_PRIMEstatic doubleDOUBLE_EPS
-
Constructor Summary
Constructors Constructor Description UtilFunctions()
-
Method Summary
All Methods Static Methods Concrete Methods Modifier and Type Method Description static StringaddTimeToDate(String dateString, int amountToAdd, String timeformat)static StringcolumnStringToCSVString(String input, String separator)static intcompareTo(Types.ValueType vt, Object in1, Object in2)static intcompareVersion(String version1, String version2)Compares two version strings of format x.y.z, where x is major, y is minor, and z is maintenance release.static longcomputeBlockIndex(long cellIndex, int blockSize)Computes the 1-based block index based on the global cell index and block size meta data.static longcomputeBlockNumber(int[] ix, long[] dims, int blen)Calculates the number of the block this index refers to (basically a linearisation).static intcomputeBlockSize(long len, long blockIndex, long blockSize)Computes the actual block size based on matrix dimension, block index, and block size meta data.static intcomputeCellInBlock(long cellIndex, int blockSize)Computes the 0-based cell-in-block index based on the global cell index and block size meta data.static longcomputeCellIndex(long blockIndex, int blockSize, int cellInBlock)Computes the global 1-based cell index based on the block index, block size meta data, and specific 0-based in-block cell index.static long[]computeNextTensorIndexes(TensorCharacteristics tc, long[] ix)Computes the next tensor indexes array.static intcomputeNnz(double[] a, int ai, int len)static intcomputeNnz(float[] a, int ai, int len)static intcomputeNnz(int[] a, int ai, int len)static intcomputeNnz(long[] a, int ai, int len)static intcomputeNnz(String[] a, int ai, int len)static intcomputeNnz(BitSet a, int ai, int len)static longcomputeNnz(SparseBlock a, int[] aix, int ai, int alen)static voidcomputeSliceInfo(TensorCharacteristics tc, long[] blockIx, int[] outDims, int[] offset)Computes the slice dimensions and offsets for the block slice of another tensor with the size given byTensorCharacteristics.static long[]computeTensorIndexes(TensorCharacteristics tc, long blockIndex)Computes the tensor indexes array given a blockIndex we ant to compute.static booleancontainsZero(double[] data, int pos, int len)static double[]convertStringToDoubleArray(String[] original)static String[]copyAsStringToArray(String[] input, Object value)static Types.ValueType[]copyOf(Types.ValueType[] schema1, Types.ValueType[] schema2)static intcountNonZeros(double[] data, int pos, int len)static StringdateFormat(long date, String outputFormat)static StringdateFormat(String dateString, String outputFormat)static StringdateFormat(String dateString, String inputFormat, String outputFormat)static ObjectdoubleToObject(Types.ValueType vt, double in)static ObjectdoubleToObject(Types.ValueType vt, double in, boolean sparse)static StringformatMemorySize(long arg)Format a memory size with g/m/k quantifiers into its number representation.static intfrequency(Types.ValueType[] schema, Types.ValueType vt)static FrameBlockgenerateRandomFrameBlock(int rows, int cols, Types.ValueType[] schema, Random random)Generates a random FrameBlock with given parameters.static ObjectgenerateRandomValueFromValueType(Types.ValueType valueType, Random random)Generates a random value for a given Value Typestatic ArrayList<Integer>getAlignedBlockSizes(int len, int k, int align)static intgetAsciiAtIdx(String s, int idx)static ArrayList<Integer>getBalancedBlockSizesDefault(int len, int k, boolean constK)static voidgetBlockBounds(TensorIndexes ix, long[] dims, int blen, int[] lower, int[] upper)static int[]getBlockSizes(int num, int numBlocks)static String[]getDominantDateFormat(String[] values)static doublegetDouble(Object obj)static intgetEndIndex(int arrayLength, int startIndex, int blockSize)static IndexRangegetSelectedRangeForZeroOut(IndexedMatrixValue in, int blen, IndexRange indexRange)static IndexRangegetSelectedRangeForZeroOut(Pair<Long,FrameBlock> in, int blen, IndexRange indexRange, long lSrcRowIndex, long lDestRowIndex)static int[]getSeqArray(int low, int up, int incr)Obtain sequence arraystatic longgetSeqLength(double from, double to, double incr)static longgetSeqLength(double from, double to, double incr, boolean check)static List<Integer>getSeqList(int low, int up, int incr)Obtain sequence liststatic int[]getSortedSampleIndexes(int range, int sampleSize)static int[]getSortedSampleIndexes(int range, int sampleSize, long seed)static String[]getSplittedStringAsArray(String input)static String[]getTimestamp(String[] values)static intintHashCode(int key1, int key2)static intintHashCodeRobust(int key1, int key2)static booleanisBoolean(String str)static StringisDateColumn(String values)static booleanisInBlockRange(MatrixIndexes ix, int blen, long rl, long ru, long cl, long cu)static booleanisInBlockRange(MatrixIndexes ix, int blen, IndexRange ixrange)static booleanisInFrameBlockRange(Long ix, int blen, long rl, long ru)static booleanisInFrameBlockRange(Long ix, int blen, IndexRange ixrange)static booleanisIntegerNumber(String str)static booleanisNonZero(Object obj)static booleanisSpecial(double value)static doublejaccardSim(String x, String y)static intlongHashCode(long key1)static intlongHashCode(long key1, long key2)Returns the hash code for a long-long pair.static intlongHashCode(long key1, long key2, long key3)Returns the hash code for a long-long-long triple.static bytemax(byte[] array)static Types.ValueType[]nCopies(int n, Types.ValueType vt)static intnextIntPow2(int in)static doubleobjectToDouble(Types.ValueType vt, Object in)static doubleobjectToDoubleSafe(Types.ValueType vt, Object in)static ObjectobjectToObject(Types.ValueType vt, Object in)static ObjectobjectToObject(Types.ValueType vt, Object in, boolean ignoreNull)static StringobjectToString(Object in)static StringobjectToString(Object in, boolean ignoreNull)Convert object to stringstatic longparseMemorySize(String arg)Parses a memory size with optional g/m/k quantifiers into its number representation.static doubleparseToDouble(String str, Set<String> isNan)Safe double parsing including handling of NAs.static intparseToInt(String str)static longparseToLong(String str)static longpow(int base, int exp)static longprod(int[] arr)static longprod(int[] arr, int off)static longprod(long[] arr)static Stringquote(String s)static introundToNext(int val, int factor)static String[]splitRecodeEntry(String s)static ObjectstringToObject(Types.ValueType vt, String in)static Types.ValueType[]stringToValueType(String[] schemaValues)Generates a ValueType array from a String arraystatic inttoInt(double val)static inttoInt(Object obj)static longtoLong(double val)static longtoMillis(String dateString)static longtoMillis(String dateString, String dateFormat)static String[]toStringArray(Object[] original)static Stringunquote(String s)
-
-
-
Field Detail
-
DOUBLE_EPS
public static final double DOUBLE_EPS
-
ADD_PRIME1
public static final long ADD_PRIME1
- See Also:
- Constant Field Values
-
DIVIDE_PRIME
public static final int DIVIDE_PRIME
- See Also:
- Constant Field Values
-
-
Method Detail
-
intHashCode
public static int intHashCode(int key1, int key2)
-
intHashCodeRobust
public static int intHashCodeRobust(int key1, int key2)
-
longHashCode
public static int longHashCode(long key1)
-
longHashCode
public static int longHashCode(long key1, long key2)Returns the hash code for a long-long pair. This is the default hash function for the keys of a distributed matrix in MR/Spark.- Parameters:
key1- first long keykey2- second long key- Returns:
- hash code
-
longHashCode
public static int longHashCode(long key1, long key2, long key3)Returns the hash code for a long-long-long triple. This is the default hash function for the keys of a distributed matrix in MR/Spark.- Parameters:
key1- first long keykey2- second long keykey3- third long key- Returns:
- hash code
-
nextIntPow2
public static int nextIntPow2(int in)
-
pow
public static long pow(int base, int exp)
-
computeBlockIndex
public static long computeBlockIndex(long cellIndex, int blockSize)Computes the 1-based block index based on the global cell index and block size meta data. See computeCellIndex for the inverse operation.- Parameters:
cellIndex- global cell indexblockSize- block size- Returns:
- 1-based block index
-
computeCellInBlock
public static int computeCellInBlock(long cellIndex, int blockSize)Computes the 0-based cell-in-block index based on the global cell index and block size meta data. See computeCellIndex for the inverse operation.- Parameters:
cellIndex- global cell indexblockSize- block size- Returns:
- 0-based cell-in-block index
-
computeCellIndex
public static long computeCellIndex(long blockIndex, int blockSize, int cellInBlock)Computes the global 1-based cell index based on the block index, block size meta data, and specific 0-based in-block cell index. NOTE: this is equivalent to cellIndexCalculation.- Parameters:
blockIndex- block indexblockSize- block sizecellInBlock- 0-based cell-in-block index- Returns:
- global 1-based cell index
-
computeBlockSize
public static int computeBlockSize(long len, long blockIndex, long blockSize)Computes the actual block size based on matrix dimension, block index, and block size meta data. For boundary blocks, the actual block size is less or equal than the block size meta data; otherwise they are identical.- Parameters:
len- matrix dimensionblockIndex- block indexblockSize- block size metadata- Returns:
- actual block size
-
computeNextTensorIndexes
public static long[] computeNextTensorIndexes(TensorCharacteristics tc, long[] ix)
Computes the next tensor indexes array.- Parameters:
tc- the tensor characteristicsix- the tensor indexes array (will be changed)- Returns:
- the tensor indexes array (changed)
-
computeTensorIndexes
public static long[] computeTensorIndexes(TensorCharacteristics tc, long blockIndex)
Computes the tensor indexes array given a blockIndex we ant to compute. Note that if a sequence of tensor indexes array will be computed, it is faster to useUtilFunctions.computeNextTensorIndexes(TensorCharacteristics,long[]).- Parameters:
tc- the tensor characteristicsblockIndex- the number of the block ([0-tc.getNumBlocks()[ valid)- Returns:
- the tensor index array
-
computeSliceInfo
public static void computeSliceInfo(TensorCharacteristics tc, long[] blockIx, int[] outDims, int[] offset)
Computes the slice dimensions and offsets for the block slice of another tensor with the size given byTensorCharacteristics.- Parameters:
tc- tensor characteristics of the block to sliceblockIx- the tensor block indexoutDims- the slice dimension sizeoffset- the offset where the slice should start
-
computeBlockNumber
public static long computeBlockNumber(int[] ix, long[] dims, int blen)Calculates the number of the block this index refers to (basically a linearisation).- Parameters:
ix- the dimensional indexesdims- length of dimensionsblen- length of blocks- Returns:
- the number of the block
-
getBalancedBlockSizesDefault
public static ArrayList<Integer> getBalancedBlockSizesDefault(int len, int k, boolean constK)
-
getAlignedBlockSizes
public static ArrayList<Integer> getAlignedBlockSizes(int len, int k, int align)
-
isInBlockRange
public static boolean isInBlockRange(MatrixIndexes ix, int blen, long rl, long ru, long cl, long cu)
-
isInFrameBlockRange
public static boolean isInFrameBlockRange(Long ix, int blen, long rl, long ru)
-
isInBlockRange
public static boolean isInBlockRange(MatrixIndexes ix, int blen, IndexRange ixrange)
-
isInFrameBlockRange
public static boolean isInFrameBlockRange(Long ix, int blen, IndexRange ixrange)
-
getSelectedRangeForZeroOut
public static IndexRange getSelectedRangeForZeroOut(IndexedMatrixValue in, int blen, IndexRange indexRange)
-
getSelectedRangeForZeroOut
public static IndexRange getSelectedRangeForZeroOut(Pair<Long,FrameBlock> in, int blen, IndexRange indexRange, long lSrcRowIndex, long lDestRowIndex)
-
parseToDouble
public static double parseToDouble(String str, Set<String> isNan)
Safe double parsing including handling of NAs. Previously, we also used this wrapper for handling thread contention in multi-threaded environments because Double.parseDouble relied on a synchronized cache (which was replaced with thread-local caches in JDK8).- Parameters:
str- string to parse to doubleisNan- collection of Nan string which if encountered should be parsed to nan value- Returns:
- double value
-
parseToInt
public static int parseToInt(String str)
-
parseToLong
public static long parseToLong(String str)
-
toInt
public static int toInt(double val)
-
toLong
public static long toLong(double val)
-
toInt
public static int toInt(Object obj)
-
getSeqLength
public static long getSeqLength(double from, double to, double incr)
-
getSeqLength
public static long getSeqLength(double from, double to, double incr, boolean check)
-
getSeqList
public static List<Integer> getSeqList(int low, int up, int incr)
Obtain sequence list- Parameters:
low- lower bound (inclusive)up- upper bound (inclusive)incr- increment- Returns:
- list of integers
-
getSeqArray
public static int[] getSeqArray(int low, int up, int incr)Obtain sequence array- Parameters:
low- lower bound (inclusive)up- upper bound (inclusive)incr- increment- Returns:
- array of integers
-
roundToNext
public static int roundToNext(int val, int factor)
-
doubleToObject
public static Object doubleToObject(Types.ValueType vt, double in)
-
doubleToObject
public static Object doubleToObject(Types.ValueType vt, double in, boolean sparse)
-
stringToObject
public static Object stringToObject(Types.ValueType vt, String in)
-
objectToDoubleSafe
public static double objectToDoubleSafe(Types.ValueType vt, Object in)
-
objectToDouble
public static double objectToDouble(Types.ValueType vt, Object in)
-
objectToString
public static String objectToString(Object in, boolean ignoreNull)
Convert object to string- Parameters:
in- objectignoreNull- If this flag has set, it will ignore null. This flag is mainly used in merge functionality to override data with "null" data.- Returns:
- string representation of object
-
objectToObject
public static Object objectToObject(Types.ValueType vt, Object in)
-
objectToObject
public static Object objectToObject(Types.ValueType vt, Object in, boolean ignoreNull)
-
compareTo
public static int compareTo(Types.ValueType vt, Object in1, Object in2)
-
compareVersion
public static int compareVersion(String version1, String version2)
Compares two version strings of format x.y.z, where x is major, y is minor, and z is maintenance release.- Parameters:
version1- first version stringversion2- second version string- Returns:
- 1 if version1 greater, -1 if version2 greater, 0 if equal
-
isBoolean
public static boolean isBoolean(String str)
-
isIntegerNumber
public static boolean isIntegerNumber(String str)
-
isSpecial
public static boolean isSpecial(double value)
-
getSortedSampleIndexes
public static int[] getSortedSampleIndexes(int range, int sampleSize)
-
getSortedSampleIndexes
public static int[] getSortedSampleIndexes(int range, int sampleSize, long seed)
-
max
public static byte max(byte[] array)
-
getAsciiAtIdx
public static int getAsciiAtIdx(String s, int idx)
-
parseMemorySize
public static long parseMemorySize(String arg)
Parses a memory size with optional g/m/k quantifiers into its number representation.- Parameters:
arg- memory size as readable string- Returns:
- byte count of memory size
-
formatMemorySize
public static String formatMemorySize(long arg)
Format a memory size with g/m/k quantifiers into its number representation.- Parameters:
arg- byte count of memory size- Returns:
- memory size as readable string
-
getDouble
public static double getDouble(Object obj)
-
isNonZero
public static boolean isNonZero(Object obj)
-
computeNnz
public static int computeNnz(double[] a, int ai, int len)
-
computeNnz
public static int computeNnz(float[] a, int ai, int len)
-
computeNnz
public static int computeNnz(long[] a, int ai, int len)
-
computeNnz
public static int computeNnz(int[] a, int ai, int len)
-
computeNnz
public static int computeNnz(BitSet a, int ai, int len)
-
computeNnz
public static int computeNnz(String[] a, int ai, int len)
-
computeNnz
public static long computeNnz(SparseBlock a, int[] aix, int ai, int alen)
-
nCopies
public static Types.ValueType[] nCopies(int n, Types.ValueType vt)
-
frequency
public static int frequency(Types.ValueType[] schema, Types.ValueType vt)
-
copyOf
public static Types.ValueType[] copyOf(Types.ValueType[] schema1, Types.ValueType[] schema2)
-
countNonZeros
public static int countNonZeros(double[] data, int pos, int len)
-
containsZero
public static boolean containsZero(double[] data, int pos, int len)
-
prod
public static long prod(long[] arr)
-
prod
public static long prod(int[] arr)
-
prod
public static long prod(int[] arr, int off)
-
getBlockBounds
public static void getBlockBounds(TensorIndexes ix, long[] dims, int blen, int[] lower, int[] upper)
-
toMillis
public static long toMillis(String dateString)
-
dateFormat
public static String dateFormat(String dateString, String inputFormat, String outputFormat)
-
addTimeToDate
public static String addTimeToDate(String dateString, int amountToAdd, String timeformat)
-
columnStringToCSVString
public static String columnStringToCSVString(String input, String separator)
-
generateRandomFrameBlock
public static FrameBlock generateRandomFrameBlock(int rows, int cols, Types.ValueType[] schema, Random random)
Generates a random FrameBlock with given parameters.- Parameters:
rows- frame rowscols- frame colsschema- frame schemarandom- random number generator- Returns:
- FrameBlock
-
generateRandomValueFromValueType
public static Object generateRandomValueFromValueType(Types.ValueType valueType, Random random)
Generates a random value for a given Value Type- Parameters:
valueType- the ValueType of which to generate the valuerandom- random number generator- Returns:
- Object
-
stringToValueType
public static Types.ValueType[] stringToValueType(String[] schemaValues)
Generates a ValueType array from a String array- Parameters:
schemaValues- the string schema of which to generate the ValueType- Returns:
- ValueType[]
-
getEndIndex
public static int getEndIndex(int arrayLength, int startIndex, int blockSize)
-
getBlockSizes
public static int[] getBlockSizes(int num, int numBlocks)
-
convertStringToDoubleArray
public static double[] convertStringToDoubleArray(String[] original)
-
-