You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
41 lines
948 B
41 lines
948 B
6 years ago
|
from __future__ import print_function, division, absolute_import
|
||
|
|
|
||
|
|
import posixpath
|
||
|
|
|
||
|
|
from .glob import generic_glob
|
||
|
|
from ..base import tokenize
|
||
|
|
|
||
|
|
import hdfs3
|
||
|
|
|
||
|
|
|
||
|
|
class HDFS3HadoopFileSystem(object):
|
||
|
|
sep = "/"
|
||
|
|
|
||
|
|
def __init__(self, **kwargs):
|
||
|
|
self.fs = hdfs3.HDFileSystem(**kwargs)
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def from_hdfs3(cls, fs):
|
||
|
|
out = object.__new__(cls)
|
||
|
|
out.fs = fs
|
||
|
|
return out
|
||
|
|
|
||
|
|
def open(self, path, mode='rb', **kwargs):
|
||
|
|
return self.fs.open(path, mode=mode, **kwargs)
|
||
|
|
|
||
|
|
def glob(self, path):
|
||
|
|
return sorted(generic_glob(self.fs, posixpath, path))
|
||
|
|
|
||
|
|
def mkdirs(self, path):
|
||
|
|
return self.fs.makedirs(path)
|
||
|
|
|
||
|
|
def ukey(self, path):
|
||
|
|
return tokenize(path, self.fs.info(path)['last_mod'])
|
||
|
|
|
||
|
|
def size(self, path):
|
||
|
|
return self.fs.info(path)['size']
|
||
|
|
|
||
|
|
def _get_pyarrow_filesystem(self):
|
||
|
|
from .pyarrow import HDFS3Wrapper
|
||
|
|
return HDFS3Wrapper(self.fs)
|