Package org.apache.sysds.hops
Class Hop
- java.lang.Object
- 
- org.apache.sysds.hops.Hop
 
- 
- All Implemented Interfaces:
- ParseInfo
 - Direct Known Subclasses:
- DataOp,- FunctionOp,- IndexingOp,- LeftIndexingOp,- LiteralOp,- MultiThreadedHop,- NaryOp
 
 public abstract class Hop extends Object implements ParseInfo 
- 
- 
Field SummaryFields Modifier and Type Field Description int_beginColumnint_beginLineint_endColumnint_endLineString_filenameString_textstatic longCPThreshold
 - 
Constructor SummaryConstructors Constructor Description Hop(String l, Types.DataType dt, Types.ValueType vt)
 - 
Method SummaryAll Methods Static Methods Instance Methods Abstract Methods Concrete Methods Modifier and Type Method Description voidactivatePrefetch()Activate prefetch of HOP.voidaddAllInputs(ArrayList<Hop> list)voidaddInput(Hop h)abstract booleanallowsAllExecTypes()voidcheckAndSetForcedPlatform()voidcheckAndSetInvalidCPDimsAndSize()abstract voidcheckArity()Check whether this Hop has a correct number of inputs.voidclearMemEstimate()abstract Objectclone()booleancolsKnown()abstract booleancompare(Hop that)longcompressedSize()doublecomputeBoundsInformation(Hop input)static doublecomputeBoundsInformation(Hop input, LocalVariableMap vars)static doublecomputeBoundsInformation(Hop input, LocalVariableMap vars, HashMap<Long,Double> memo)voidcomputeMemEstimate(MemoTable memo)Computes the estimate of memory required to store the input/output of this hop in memory.static longcomputeSizeInformation(Hop input)longcomputeSizeInformation(Hop input, LocalVariableMap vars)longcomputeSizeInformation(Hop input, LocalVariableMap vars, HashMap<Long,Long> memo)voidconstructAndSetLopsDataFlowProperties()abstract LopconstructLops()static LopcreateOffsetLop(Hop hop, boolean repCols)voiddeactivatePrefetch()booleandimsKnown()booleandimsKnown(boolean includeNnz)booleandimsKnownAny()booleanfederatedCostInitialized()Check if federated cost has been initialized for this Hop.intgetBeginColumn()intgetBeginLine()intgetBlocksize()longgetCompressedSize()DataCharacteristicsgetDataCharacteristics()Types.DataTypegetDataType()longgetDim(int i)longgetDim1()Get the number of rows in the Hop.longgetDim2()Get the number of columns in the Hop.intgetEndColumn()intgetEndLine()Types.ExecTypegetExecType()FederatedCostgetFederatedCost()FEDInstruction.FederatedOutputgetFederatedOutput()StringgetFilename()Types.ExecTypegetForcedExecType()longgetHopID()ArrayList<Hop>getInput()HopgetInput(int ix)doublegetInputMemEstimate()Get the memory estimate of inputs as the sum of input estimates in bytes.doublegetInputMemEstimate(double injectedDefault)Get the memory estimate of inputs as the sum of input estimates in bytes.doublegetInputOutputSize(Collection<String> exclVars)doublegetIntermediateMemEstimate()longgetLength()LopgetLops()doublegetMemEstimate()NOTES: * Purpose: Whenever the output dimensions / sparsity of a hop are unknown, this hop should store its worst-case output statistics (if known) in that table.StringgetName()longgetNnz()abstract StringgetOpString()doublegetOutputMemEstimate()Output memory estimate in bytes.doublegetOutputMemEstimate(double injectedDefault)Output memory estimate in bytes with negative memory estimates replaced by the injected default.ArrayList<Hop>getParent()PrivacyConstraintgetPrivacy()doublegetRepetitions()doublegetSparsity()doublegetSpBroadcastSize()StringgetText()MatrixObject.UpdateTypegetUpdateType()Types.ValueTypegetValueType()booleanhasCompressedInput()booleanhasFederatedOutput()booleanhasLocalOutput()booleanhasMatrixInputWithDifferentBlocksizes()booleanhasValidCPDimsAndSize()booleanisCompressedOutput()booleanisFederated()Checks if ExecType is federated.booleanisFederatedDataOp()Checks if the hop is a DataOp with federated data.abstract booleanisGPUEnabled()In memory-based optimizer mode (see OptimizerUtils.isMemoryBasedOptLevel()), the exectype is determined by checking this method as well as memory budget of this Hop.booleanisMatrix()booleanisMemEstimated()booleanisOutputEmptyBlocks()booleanisRequiredDecompression()booleanisScalar()booleanisTransposeSafe()Defines if this operation is transpose-safe, which means that the result of op(input) is equivalent to op(t(input)).booleanisVisited()booleanprefetchActivated()Checks if prefetch is activated for this hop.StringprintErrorLocation()voidrefreshColsParameterInformation(Hop input, LocalVariableMap vars)voidrefreshColsParameterInformation(Hop input, LocalVariableMap vars, HashMap<Long,Long> memo)voidrefreshMemEstimates(MemoTable memo)Recursively computes memory estimates for all the Hops in the DAG rooted at the current hop pointed bythis.voidrefreshRowsParameterInformation(Hop input, LocalVariableMap vars)voidrefreshRowsParameterInformation(Hop input, LocalVariableMap vars, HashMap<Long,Long> memo)abstract voidrefreshSizeInformation()Update the output size information for this hop.booleanrequiresCheckpoint()booleanrequiresCompression()booleanrequiresLineageCaching()booleanrequiresReblock()booleanrequiresRecompile()Indicates if dynamic recompilation is required for this hop.voidresetExecType()voidresetFederatedCost()Reset federated cost of this hop and all children of this hop.static voidresetRecompilationFlag(ArrayList<Hop> hops, Types.ExecType et, Recompiler.ResetType reset)static voidresetRecompilationFlag(Hop hops, Types.ExecType et, Recompiler.ResetType reset)HopresetVisitStatus()static voidresetVisitStatus(ArrayList<Hop> hops)static voidresetVisitStatus(ArrayList<Hop> hops, boolean force)voidresetVisitStatusForced(HashSet<Long> memo)booleanrowsKnown()voidsetBeginColumn(int passed)voidsetBeginLine(int passed)voidsetBlocksize(int blen)voidsetCompressedOutput(boolean value)voidsetCompressedSize(long size)voidsetDataType(Types.DataType dt)voidsetDim(int i, long dim)voidsetDim1(long dim1)voidsetDim2(long dim2)voidsetEndColumn(int passed)voidsetEndLine(int passed)voidsetExecType(Types.ExecType execType)voidsetFederatedCost(FederatedCost cost)voidsetFederatedOutput(FEDInstruction.FederatedOutput federatedOutput)voidsetFilename(String passed)voidsetForcedExecType(Types.ExecType etype)voidsetLops(Lop lops)voidsetMemEstimate(double mem)Sets memory estimate in bytesvoidsetName(String _name)voidsetNnz(long nnz)voidsetOutputEmptyBlocks(boolean flag)voidsetParseInfo(ParseInfo parseInfo)Set parse information.voidsetPrivacy(PrivacyConstraint privacy)voidsetRequiresCheckpoint(boolean flag)voidsetRequiresCompression()voidsetRequiresCompression(AWTreeNode node)voidsetRequiresDeCompression()voidsetRequiresLineageCaching(boolean flag)voidsetRequiresReblock(boolean flag)voidsetRequiresRecompile()Marks the hop for dynamic recompilation.voidsetText(String text)voidsetUpdateType(MatrixObject.UpdateType update)voidsetValueType(Types.ValueType vt)voidsetVisited()voidsetVisited(boolean flag)booleansomeInputFederated()StringtoString()voidupdateLopFedOut(Lop lop)voidupdateLopFedOut(Lop lop, Types.ExecType execType, FEDInstruction.FederatedOutput fedOut)voidupdateRepetitionEstimates(double repetitions)
 
- 
- 
- 
Field Detail- 
CPThresholdpublic static final long CPThreshold - See Also:
- Constant Field Values
 
 - 
_beginLinepublic int _beginLine 
 - 
_beginColumnpublic int _beginColumn 
 - 
_endLinepublic int _endLine 
 - 
_endColumnpublic int _endColumn 
 - 
_filenamepublic String _filename 
 - 
_textpublic String _text 
 
- 
 - 
Constructor Detail- 
Hoppublic Hop(String l, Types.DataType dt, Types.ValueType vt) 
 
- 
 - 
Method Detail- 
getHopIDpublic long getHopID() 
 - 
checkAritypublic abstract void checkArity() Check whether this Hop has a correct number of inputs. (Some Hops can have a variable number of inputs, such as DataOp, DataGenOp, ParameterizedBuiltinOp, ReorgOp, TernaryOp, QuaternaryOp, MultipleOp, DnnOp, and SpoofFusedOp.) Parameterized Hops (such as DataOp) can check that the number of parameters matches the number of inputs.
 - 
getExecTypepublic Types.ExecType getExecType() 
 - 
setExecTypepublic void setExecType(Types.ExecType execType) 
 - 
setFederatedOutputpublic void setFederatedOutput(FEDInstruction.FederatedOutput federatedOutput) 
 - 
activatePrefetchpublic void activatePrefetch() Activate prefetch of HOP.
 - 
deactivatePrefetchpublic void deactivatePrefetch() 
 - 
prefetchActivatedpublic boolean prefetchActivated() Checks if prefetch is activated for this hop.- Returns:
- true if prefetch is activated
 
 - 
resetExecTypepublic void resetExecType() 
 - 
getForcedExecTypepublic Types.ExecType getForcedExecType() 
 - 
setForcedExecTypepublic void setForcedExecType(Types.ExecType etype) 
 - 
allowsAllExecTypespublic abstract boolean allowsAllExecTypes() 
 - 
isTransposeSafepublic boolean isTransposeSafe() Defines if this operation is transpose-safe, which means that the result of op(input) is equivalent to op(t(input)). Usually, this applies to aggregate operations with fixed output dimension. Finally, this information is very useful in order to safely optimize the plan for sparse vectors, which otherwise would be (currently) always represented dense.- Returns:
- always returns false
 
 - 
checkAndSetForcedPlatformpublic void checkAndSetForcedPlatform() 
 - 
checkAndSetInvalidCPDimsAndSizepublic void checkAndSetInvalidCPDimsAndSize() 
 - 
hasValidCPDimsAndSizepublic boolean hasValidCPDimsAndSize() 
 - 
hasMatrixInputWithDifferentBlocksizespublic boolean hasMatrixInputWithDifferentBlocksizes() 
 - 
setRequiresReblockpublic void setRequiresReblock(boolean flag) 
 - 
requiresReblockpublic boolean requiresReblock() 
 - 
setRequiresCheckpointpublic void setRequiresCheckpoint(boolean flag) 
 - 
requiresCheckpointpublic boolean requiresCheckpoint() 
 - 
setRequiresCompressionpublic void setRequiresCompression() 
 - 
setRequiresCompressionpublic void setRequiresCompression(AWTreeNode node) 
 - 
setRequiresDeCompressionpublic void setRequiresDeCompression() 
 - 
isRequiredDecompressionpublic boolean isRequiredDecompression() 
 - 
requiresCompressionpublic boolean requiresCompression() 
 - 
setCompressedOutputpublic void setCompressedOutput(boolean value) 
 - 
setCompressedSizepublic void setCompressedSize(long size) 
 - 
getCompressedSizepublic long getCompressedSize() 
 - 
isCompressedOutputpublic boolean isCompressedOutput() 
 - 
hasCompressedInputpublic boolean hasCompressedInput() 
 - 
compressedSizepublic long compressedSize() 
 - 
setRequiresLineageCachingpublic void setRequiresLineageCaching(boolean flag) 
 - 
requiresLineageCachingpublic boolean requiresLineageCaching() 
 - 
updateLopFedOutpublic void updateLopFedOut(Lop lop) 
 - 
updateLopFedOutpublic void updateLopFedOut(Lop lop, Types.ExecType execType, FEDInstruction.FederatedOutput fedOut) 
 - 
constructAndSetLopsDataFlowPropertiespublic void constructAndSetLopsDataFlowProperties() 
 - 
setOutputEmptyBlockspublic void setOutputEmptyBlocks(boolean flag) 
 - 
isOutputEmptyBlockspublic boolean isOutputEmptyBlocks() 
 - 
getInputOutputSizepublic double getInputOutputSize(Collection<String> exclVars) 
 - 
getMemEstimatepublic double getMemEstimate() NOTES: * Purpose: Whenever the output dimensions / sparsity of a hop are unknown, this hop should store its worst-case output statistics (if known) in that table. * Invocation: Intended to be called for ALL root nodes of one Hops DAG with the same (initially empty) memo table.- Returns:
- memory estimate in bytes
 
 - 
setMemEstimatepublic void setMemEstimate(double mem) Sets memory estimate in bytes- Parameters:
- mem- memory estimate
 
 - 
clearMemEstimatepublic void clearMemEstimate() 
 - 
isMemEstimatedpublic boolean isMemEstimated() 
 - 
getInputMemEstimatepublic double getInputMemEstimate() Get the memory estimate of inputs as the sum of input estimates in bytes.- Returns:
- input memory estimate in bytes
 
 - 
getInputMemEstimatepublic double getInputMemEstimate(double injectedDefault) Get the memory estimate of inputs as the sum of input estimates in bytes.- Parameters:
- injectedDefault- default memory estimate (bytes) used when the memory estimate of the input is negative
- Returns:
- input memory estimate in bytes
 
 - 
getOutputMemEstimatepublic double getOutputMemEstimate() Output memory estimate in bytes.- Returns:
- output memory estimate in bytes
 
 - 
getOutputMemEstimatepublic double getOutputMemEstimate(double injectedDefault) Output memory estimate in bytes with negative memory estimates replaced by the injected default. The injected default represents the memory estimate per output cell, hence it is multiplied by the estimated dimensions of the output of the hop.- Parameters:
- injectedDefault- memory estimate to be returned in case the memory estimate defaults to a negative number
- Returns:
- output memory estimate in bytes
 
 - 
getIntermediateMemEstimatepublic double getIntermediateMemEstimate() 
 - 
getSpBroadcastSizepublic double getSpBroadcastSize() 
 - 
computeMemEstimatepublic void computeMemEstimate(MemoTable memo) Computes the estimate of memory required to store the input/output of this hop in memory. This is the default implementation (orchestration of hop-specific implementation) that should suffice for most hops. If a hop requires more control, this method should be overwritten with awareness of (1) output estimates, and (2) propagation of worst-case matrix characteristics (dimensions, sparsity). TODO remove memo table and, on constructor refresh, inference in refresh, single compute mem, maybe general computeMemEstimate, flags to indicate if estimate or not.- Parameters:
- memo- memory table
 
 - 
refreshMemEstimatespublic void refreshMemEstimates(MemoTable memo) Recursively computes memory estimates for all the Hops in the DAG rooted at the current hop pointed bythis.- Parameters:
- memo- memory table
 
 - 
isFederatedpublic boolean isFederated() Checks if ExecType is federated.- Returns:
- true if ExecType is federated
 
 - 
someInputFederatedpublic boolean someInputFederated() 
 - 
isFederatedDataOppublic boolean isFederatedDataOp() Checks if the hop is a DataOp with federated data.- Returns:
- true if hop is a federated DataOp
 
 - 
getInputpublic Hop getInput(int ix) 
 - 
addInputpublic void addInput(Hop h) 
 - 
getBlocksizepublic int getBlocksize() 
 - 
setBlocksizepublic void setBlocksize(int blen) 
 - 
setNnzpublic void setNnz(long nnz) 
 - 
getNnzpublic long getNnz() 
 - 
setPrivacypublic void setPrivacy(PrivacyConstraint privacy) 
 - 
getPrivacypublic PrivacyConstraint getPrivacy() 
 - 
getFederatedOutputpublic FEDInstruction.FederatedOutput getFederatedOutput() 
 - 
hasFederatedOutputpublic boolean hasFederatedOutput() 
 - 
hasLocalOutputpublic boolean hasLocalOutput() 
 - 
federatedCostInitializedpublic boolean federatedCostInitialized() Check if federated cost has been initialized for this Hop.- Returns:
- true if federated cost has been initialized
 
 - 
getFederatedCostpublic FederatedCost getFederatedCost() 
 - 
setFederatedCostpublic void setFederatedCost(FederatedCost cost) 
 - 
resetFederatedCostpublic void resetFederatedCost() Reset federated cost of this hop and all children of this hop.
 - 
setUpdateTypepublic void setUpdateType(MatrixObject.UpdateType update) 
 - 
getUpdateTypepublic MatrixObject.UpdateType getUpdateType() 
 - 
constructLopspublic abstract Lop constructLops() 
 - 
getOpStringpublic abstract String getOpString() 
 - 
isGPUEnabledpublic abstract boolean isGPUEnabled() In memory-based optimizer mode (see OptimizerUtils.isMemoryBasedOptLevel()), the exectype is determined by checking this method as well as memory budget of this Hop. Please see findExecTypeByMemEstimate for more detail. This method is necessary because not all operator are supported efficiently on GPU (for example: operations on frames and scalar as well as operations such as table).- Returns:
- true if the Hop is eligible for GPU Exectype.
 
 - 
dimsKnownpublic boolean dimsKnown() 
 - 
dimsKnownpublic boolean dimsKnown(boolean includeNnz) 
 - 
dimsKnownAnypublic boolean dimsKnownAny() 
 - 
rowsKnownpublic boolean rowsKnown() 
 - 
colsKnownpublic boolean colsKnown() 
 - 
resetVisitStatuspublic Hop resetVisitStatus() 
 - 
resetRecompilationFlagpublic static void resetRecompilationFlag(ArrayList<Hop> hops, Types.ExecType et, Recompiler.ResetType reset) 
 - 
resetRecompilationFlagpublic static void resetRecompilationFlag(Hop hops, Types.ExecType et, Recompiler.ResetType reset) 
 - 
getDim1public long getDim1() Get the number of rows in the Hop.- Returns:
- a long.
 
 - 
setDim1public void setDim1(long dim1) 
 - 
getDim2public long getDim2() Get the number of columns in the Hop.- Returns:
- a long.
 
 - 
setDim2public void setDim2(long dim2) 
 - 
getDimpublic long getDim(int i) 
 - 
setDimpublic void setDim(int i, long dim)
 - 
getLengthpublic long getLength() 
 - 
getSparsitypublic double getSparsity() 
 - 
getDataCharacteristicspublic DataCharacteristics getDataCharacteristics() 
 - 
getLopspublic Lop getLops() 
 - 
setLopspublic void setLops(Lop lops) 
 - 
isVisitedpublic boolean isVisited() 
 - 
getDataTypepublic Types.DataType getDataType() 
 - 
setDataTypepublic void setDataType(Types.DataType dt) 
 - 
isScalarpublic boolean isScalar() 
 - 
isMatrixpublic boolean isMatrix() 
 - 
setVisitedpublic void setVisited() 
 - 
setVisitedpublic void setVisited(boolean flag) 
 - 
setNamepublic void setName(String _name) 
 - 
getNamepublic String getName() 
 - 
getValueTypepublic Types.ValueType getValueType() 
 - 
setValueTypepublic void setValueType(Types.ValueType vt) 
 - 
requiresRecompilepublic boolean requiresRecompile() Indicates if dynamic recompilation is required for this hop.- Returns:
- true if dynamic recompilation required
 
 - 
setRequiresRecompilepublic void setRequiresRecompile() Marks the hop for dynamic recompilation.
 - 
refreshSizeInformationpublic abstract void refreshSizeInformation() Update the output size information for this hop.
 - 
computeSizeInformationpublic static long computeSizeInformation(Hop input) 
 - 
refreshRowsParameterInformationpublic void refreshRowsParameterInformation(Hop input, LocalVariableMap vars) 
 - 
refreshRowsParameterInformationpublic void refreshRowsParameterInformation(Hop input, LocalVariableMap vars, HashMap<Long,Long> memo) 
 - 
refreshColsParameterInformationpublic void refreshColsParameterInformation(Hop input, LocalVariableMap vars) 
 - 
refreshColsParameterInformationpublic void refreshColsParameterInformation(Hop input, LocalVariableMap vars, HashMap<Long,Long> memo) 
 - 
computeSizeInformationpublic long computeSizeInformation(Hop input, LocalVariableMap vars) 
 - 
computeSizeInformationpublic long computeSizeInformation(Hop input, LocalVariableMap vars, HashMap<Long,Long> memo) 
 - 
computeBoundsInformationpublic double computeBoundsInformation(Hop input) 
 - 
computeBoundsInformationpublic static double computeBoundsInformation(Hop input, LocalVariableMap vars) 
 - 
computeBoundsInformationpublic static double computeBoundsInformation(Hop input, LocalVariableMap vars, HashMap<Long,Double> memo) 
 - 
updateRepetitionEstimatespublic void updateRepetitionEstimates(double repetitions) 
 - 
getRepetitionspublic double getRepetitions() 
 - 
clonepublic abstract Object clone() throws CloneNotSupportedException - Throws:
- CloneNotSupportedException
 
 - 
comparepublic abstract boolean compare(Hop that) 
 - 
setBeginLinepublic void setBeginLine(int passed) - Specified by:
- setBeginLinein interface- ParseInfo
 
 - 
setBeginColumnpublic void setBeginColumn(int passed) - Specified by:
- setBeginColumnin interface- ParseInfo
 
 - 
setEndLinepublic void setEndLine(int passed) - Specified by:
- setEndLinein interface- ParseInfo
 
 - 
setEndColumnpublic void setEndColumn(int passed) - Specified by:
- setEndColumnin interface- ParseInfo
 
 - 
setFilenamepublic void setFilename(String passed) - Specified by:
- setFilenamein interface- ParseInfo
 
 - 
getBeginLinepublic int getBeginLine() - Specified by:
- getBeginLinein interface- ParseInfo
 
 - 
getBeginColumnpublic int getBeginColumn() - Specified by:
- getBeginColumnin interface- ParseInfo
 
 - 
getEndLinepublic int getEndLine() - Specified by:
- getEndLinein interface- ParseInfo
 
 - 
getEndColumnpublic int getEndColumn() - Specified by:
- getEndColumnin interface- ParseInfo
 
 - 
getFilenamepublic String getFilename() - Specified by:
- getFilenamein interface- ParseInfo
 
 - 
printErrorLocationpublic String printErrorLocation() 
 - 
setParseInfopublic void setParseInfo(ParseInfo parseInfo) Set parse information.- Parameters:
- parseInfo- parse information, such as beginning line position, beginning column position, ending line position, ending column position, text, and filename
 
 
- 
 
-