DistributedCacheFile

The distributed cache is simply hadoop's method for allowing each node local access to a specific file. The registration of that file must be called with the Configuration of the job, and not when it's on a mapper or reducer. Additionally, a unique name for the node-local access path must be used to prevent collisions in the cluster. This class provides this functionality.

In the configuration phase, the file URI is used to construct an UncachedFile instance. The name of the symlink to use on the mappers is only available after calling the add() method, which registers the file and computes the unique symlink name and returns a CachedFile instance. The CachedFile instance is Serializable, it's designed to be assigned to a val and accessed later.

The local symlink is available thorugh .file or .path depending on what type you need.

example:

class YourJob(args: Args) extends Job(args) {
  val theCachedFile = DistributedCacheFile("hdfs://ur-namenode/path/to/your/file.txt")

  def somethingThatUsesTheCachedFile() {
    doSomethingWith(theCachedFile.path) // or theCachedFile.file
  }
}

Source: DistributedCacheFile.scala

Linear Supertypes

AnyRef, Any

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def apply(path: String)(implicit mode: Mode): CachedFile
def apply(uri: URI)(implicit mode: Mode): CachedFile

Create an object that can be used to register a given URI (representing an hdfs file) that should be added to the DistributedCache.
Create an object that can be used to register a given URI (representing an hdfs file) that should be added to the DistributedCache.
uri
The fully qualified URI that points to the hdfs file to add
returns
A CachedFile instance
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def symlinkNameFor(uri: URI): String
def symlinkedUriFor(sourceUri: URI): URI
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package filecache

object DistributedCacheFile

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def apply(path: String)(implicit mode: Mode): CachedFile

def apply(uri: URI)(implicit mode: Mode): CachedFile

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def symlinkNameFor(uri: URI): String

def symlinkedUriFor(sourceUri: URI): URI

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from AnyRef

Inherited from Any

Ungrouped