Package org.apache.sysds.runtime.util
Class HDFSTool
- java.lang.Object
- 
- org.apache.sysds.runtime.util.HDFSTool
 
- 
 public class HDFSTool extends Object 
- 
- 
Field SummaryFields Modifier and Type Field Description static StringDFS_BLOCKSIZEstatic StringDFS_PERMISSIONS_ENABLEDstatic StringDFS_REPLICATIONstatic StringFS_DEFAULTFSstatic StringIO_FILE_BUFFER_SIZEstatic StringIO_SERIALIZATIONSstatic booleanUSE_BINARYBLOCK_SERIALIZATION
 - 
Constructor SummaryConstructors Constructor Description HDFSTool()
 - 
Method SummaryAll Methods Static Methods Concrete Methods Modifier and Type Method Description static voidaddBinaryBlockSerializationFramework(org.apache.hadoop.conf.Configuration job)static voidcopyFileOnHDFS(String originalDir, String newDir)static voidcreateDirIfNotExistOnHDFS(String dir, String permissions)static voidcreateDirIfNotExistOnHDFS(org.apache.hadoop.fs.Path path, String permissions)static voiddeleteFileIfExistOnHDFS(String dir)static voiddeleteFileIfExistOnHDFS(org.apache.hadoop.fs.Path outpath, org.apache.hadoop.mapred.JobConf job)static voiddeleteFileIfExistOnLFS(org.apache.hadoop.fs.Path outpath, org.apache.hadoop.mapred.JobConf job)static voiddeleteFileWithMTDIfExistOnHDFS(String fname)static longestimateNnzBasedOnFileSize(org.apache.hadoop.fs.Path path, long rlen, long clen, int blen, double factor)static booleanexistsFileOnHDFS(String fname)static org.apache.hadoop.fs.FileStatus[]getDirectoryListing(String fname)static longgetFilesizeOnHDFS(org.apache.hadoop.fs.Path path)Returns the size of a file or directory on hdfs in bytes.static org.apache.hadoop.fs.FSDataOutputStreamgetHDFSDataOutputStream(String filename, boolean overwrite)static booleanisDirectory(String fname)static booleanisFileEmpty(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir)static booleanisHDFSFileEmpty(String dir)static voidmergeIntoSingleFile(String originalDir, String newFile)static StringmetaDataToString(Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties, PrivacyConstraint privacyConstraint)static booleanreadBooleanFromHDFSFile(String filename)static double[]readColumnVectorFromHDFS(String dir, Types.FileFormat fmt, long rlen, long clen, int blen)static doublereadDoubleFromHDFSFile(String filename)static longreadIntegerFromHDFSFile(String filename)static double[][]readMatrixFromHDFS(String dir, Types.FileFormat fmt, long rlen, long clen, int blen)static ObjectreadObjectFromHDFSFile(String filename, Types.ValueType vt)static ScalarObjectreadScalarObjectFromHDFSFile(String fname, Types.ValueType vt)static StringreadStringFromHDFSFile(String filename)static voidrenameFileOnHDFS(String originalDir, String newDir)static voidwriteBooleanToHDFS(boolean b, String filename)static voidwriteDoubleToHDFS(double d, String filename)static voidwriteIntToHDFS(long i, String filename)static voidwriteMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics mc, Types.FileFormat fmt)static voidwriteMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties)static voidwriteMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties, PrivacyConstraint privacyConstraint)static voidwriteMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics mc, Types.FileFormat fmt, PrivacyConstraint privacyConstraint)static voidwriteMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics mc, Types.FileFormat fmt)static voidwriteMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties)static voidwriteMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties, PrivacyConstraint privacyConstraint)static voidwriteMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics mc, Types.FileFormat fmt, PrivacyConstraint privacyConstraint)static voidwriteObjectToHDFS(Object obj, String filename)static voidwriteScalarMetaDataFile(String mtdfile, Types.ValueType vt)static voidwriteScalarMetaDataFile(String mtdfile, Types.ValueType vt, PrivacyConstraint privacyConstraint)static voidwriteScalarToHDFS(ScalarObject scalar, String fname)Helper function to write scalars to HDFS, including writing its meta data and removing CRC files in local file systemstatic voidwriteStringToHDFS(String s, String filename)
 
- 
- 
- 
Field Detail- 
DFS_REPLICATIONpublic static final String DFS_REPLICATION - See Also:
- Constant Field Values
 
 - 
IO_FILE_BUFFER_SIZEpublic static final String IO_FILE_BUFFER_SIZE - See Also:
- Constant Field Values
 
 - 
IO_SERIALIZATIONSpublic static final String IO_SERIALIZATIONS - See Also:
- Constant Field Values
 
 - 
DFS_BLOCKSIZEpublic static final String DFS_BLOCKSIZE - See Also:
- Constant Field Values
 
 - 
DFS_PERMISSIONS_ENABLEDpublic static final String DFS_PERMISSIONS_ENABLED - See Also:
- Constant Field Values
 
 - 
FS_DEFAULTFSpublic static final String FS_DEFAULTFS - See Also:
- Constant Field Values
 
 - 
USE_BINARYBLOCK_SERIALIZATIONpublic static final boolean USE_BINARYBLOCK_SERIALIZATION - See Also:
- Constant Field Values
 
 
- 
 - 
Method Detail- 
addBinaryBlockSerializationFrameworkpublic static void addBinaryBlockSerializationFramework(org.apache.hadoop.conf.Configuration job) 
 - 
existsFileOnHDFSpublic static boolean existsFileOnHDFS(String fname) 
 - 
isDirectorypublic static boolean isDirectory(String fname) 
 - 
getDirectoryListingpublic static org.apache.hadoop.fs.FileStatus[] getDirectoryListing(String fname) 
 - 
deleteFileWithMTDIfExistOnHDFSpublic static void deleteFileWithMTDIfExistOnHDFS(String fname) throws IOException - Throws:
- IOException
 
 - 
deleteFileIfExistOnHDFSpublic static void deleteFileIfExistOnHDFS(String dir) throws IOException - Throws:
- IOException
 
 - 
deleteFileIfExistOnHDFSpublic static void deleteFileIfExistOnHDFS(org.apache.hadoop.fs.Path outpath, org.apache.hadoop.mapred.JobConf job) throws IOException- Throws:
- IOException
 
 - 
deleteFileIfExistOnLFSpublic static void deleteFileIfExistOnLFS(org.apache.hadoop.fs.Path outpath, org.apache.hadoop.mapred.JobConf job) throws IOException- Throws:
- IOException
 
 - 
isHDFSFileEmptypublic static boolean isHDFSFileEmpty(String dir) throws IOException - Throws:
- IOException
 
 - 
isFileEmptypublic static boolean isFileEmpty(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path dir) throws IOException- Throws:
- IOException
 
 - 
renameFileOnHDFSpublic static void renameFileOnHDFS(String originalDir, String newDir) throws IOException - Throws:
- IOException
 
 - 
mergeIntoSingleFilepublic static void mergeIntoSingleFile(String originalDir, String newFile) throws IOException - Throws:
- IOException
 
 - 
copyFileOnHDFSpublic static void copyFileOnHDFS(String originalDir, String newDir) throws IOException - Throws:
- IOException
 
 - 
estimateNnzBasedOnFileSizepublic static long estimateNnzBasedOnFileSize(org.apache.hadoop.fs.Path path, long rlen, long clen, int blen, double factor) throws IOException- Throws:
- IOException
 
 - 
getFilesizeOnHDFSpublic static long getFilesizeOnHDFS(org.apache.hadoop.fs.Path path) throws IOExceptionReturns the size of a file or directory on hdfs in bytes.- Parameters:
- path- file system path
- Returns:
- file size
- Throws:
- IOException- if IOException occurs
 
 - 
readDoubleFromHDFSFilepublic static double readDoubleFromHDFSFile(String filename) throws IOException - Throws:
- IOException
 
 - 
readIntegerFromHDFSFilepublic static long readIntegerFromHDFSFile(String filename) throws IOException - Throws:
- IOException
 
 - 
readBooleanFromHDFSFilepublic static boolean readBooleanFromHDFSFile(String filename) throws IOException - Throws:
- IOException
 
 - 
readStringFromHDFSFilepublic static String readStringFromHDFSFile(String filename) throws IOException - Throws:
- IOException
 
 - 
readObjectFromHDFSFilepublic static Object readObjectFromHDFSFile(String filename, Types.ValueType vt) throws IOException - Throws:
- IOException
 
 - 
readScalarObjectFromHDFSFilepublic static ScalarObject readScalarObjectFromHDFSFile(String fname, Types.ValueType vt) 
 - 
writeScalarToHDFSpublic static void writeScalarToHDFS(ScalarObject scalar, String fname) Helper function to write scalars to HDFS, including writing its meta data and removing CRC files in local file system- Parameters:
- scalar- scalar data object
- fname- file name
 
 - 
writeDoubleToHDFSpublic static void writeDoubleToHDFS(double d, String filename) throws IOException- Throws:
- IOException
 
 - 
writeIntToHDFSpublic static void writeIntToHDFS(long i, String filename) throws IOException- Throws:
- IOException
 
 - 
writeBooleanToHDFSpublic static void writeBooleanToHDFS(boolean b, String filename) throws IOException- Throws:
- IOException
 
 - 
writeStringToHDFSpublic static void writeStringToHDFS(String s, String filename) throws IOException - Throws:
- IOException
 
 - 
writeObjectToHDFSpublic static void writeObjectToHDFS(Object obj, String filename) throws IOException - Throws:
- IOException
 
 - 
writeMetaDataFilepublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics mc, Types.FileFormat fmt) throws IOException - Throws:
- IOException
 
 - 
writeMetaDataFilepublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics mc, Types.FileFormat fmt, PrivacyConstraint privacyConstraint) throws IOException - Throws:
- IOException
 
 - 
writeMetaDataFilepublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics mc, Types.FileFormat fmt) throws IOException - Throws:
- IOException
 
 - 
writeMetaDataFilepublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics mc, Types.FileFormat fmt, PrivacyConstraint privacyConstraint) throws IOException - Throws:
- IOException
 
 - 
writeMetaDataFilepublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties) throws IOException - Throws:
- IOException
 
 - 
writeMetaDataFilepublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties, PrivacyConstraint privacyConstraint) throws IOException - Throws:
- IOException
 
 - 
writeMetaDataFilepublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties) throws IOException - Throws:
- IOException
 
 - 
writeMetaDataFilepublic static void writeMetaDataFile(String mtdfile, Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties, PrivacyConstraint privacyConstraint) throws IOException - Throws:
- IOException
 
 - 
writeScalarMetaDataFilepublic static void writeScalarMetaDataFile(String mtdfile, Types.ValueType vt) throws IOException - Throws:
- IOException
 
 - 
writeScalarMetaDataFilepublic static void writeScalarMetaDataFile(String mtdfile, Types.ValueType vt, PrivacyConstraint privacyConstraint) throws IOException - Throws:
- IOException
 
 - 
metaDataToStringpublic static String metaDataToString(Types.ValueType vt, Types.ValueType[] schema, Types.DataType dt, DataCharacteristics dc, Types.FileFormat fmt, FileFormatProperties formatProperties, PrivacyConstraint privacyConstraint) throws org.apache.wink.json4j.JSONException, DMLRuntimeException - Throws:
- org.apache.wink.json4j.JSONException
- DMLRuntimeException
 
 - 
readMatrixFromHDFSpublic static double[][] readMatrixFromHDFS(String dir, Types.FileFormat fmt, long rlen, long clen, int blen) throws IOException, DMLRuntimeException - Throws:
- IOException
- DMLRuntimeException
 
 - 
readColumnVectorFromHDFSpublic static double[] readColumnVectorFromHDFS(String dir, Types.FileFormat fmt, long rlen, long clen, int blen) throws IOException, DMLRuntimeException - Throws:
- IOException
- DMLRuntimeException
 
 - 
createDirIfNotExistOnHDFSpublic static void createDirIfNotExistOnHDFS(String dir, String permissions) throws IOException - Throws:
- IOException
 
 - 
createDirIfNotExistOnHDFSpublic static void createDirIfNotExistOnHDFS(org.apache.hadoop.fs.Path path, String permissions) throws IOException- Throws:
- IOException
 
 - 
getHDFSDataOutputStreampublic static org.apache.hadoop.fs.FSDataOutputStream getHDFSDataOutputStream(String filename, boolean overwrite) throws IOException - Throws:
- IOException
 
 
- 
 
-