aboutsummaryrefslogtreecommitdiffstats
path: root/src/main/java/parallelai/spyglass/base
diff options
context:
space:
mode:
authorcra14 <chandan.rajah2@bskyb.com>2013-04-26 12:47:12 +0100
committercra14 <chandan.rajah2@bskyb.com>2013-04-26 12:47:12 +0100
commitcbf6c2903bfd0a5fe528c54382ea791c45637ded (patch)
tree2ca67f31c4d0c1779c163cb48234e821616ec6e1 /src/main/java/parallelai/spyglass/base
parentd6d712287b2bcd74f0c5bbc3ecbb106741443d7c (diff)
downloadSpyGlass-cbf6c2903bfd0a5fe528c54382ea791c45637ded.tar.gz
SpyGlass-cbf6c2903bfd0a5fe528c54382ea791c45637ded.zip
First public release of Spy Glass code base
Diffstat (limited to 'src/main/java/parallelai/spyglass/base')
-rw-r--r--src/main/java/parallelai/spyglass/base/JobLibLoader.java70
1 files changed, 70 insertions, 0 deletions
diff --git a/src/main/java/parallelai/spyglass/base/JobLibLoader.java b/src/main/java/parallelai/spyglass/base/JobLibLoader.java
new file mode 100644
index 0000000..af5bdf4
--- /dev/null
+++ b/src/main/java/parallelai/spyglass/base/JobLibLoader.java
@@ -0,0 +1,70 @@
+package parallelai.spyglass.base;
+
+import org.apache.log4j.Logger;
+import org.apache.log4j.LogManager;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+
+public class JobLibLoader {
+
+ private static Logger logger = LogManager.getLogger(JobLibLoader.class);
+
+ public static void loadJars(String libPathStr, Configuration config) {
+
+
+ try {
+ Path libPath = new Path(libPathStr);
+
+ FileSystem fs = FileSystem.get(config);
+
+ RemoteIterator<LocatedFileStatus> itr = fs.listFiles(libPath, true);
+
+ while (itr.hasNext()) {
+ LocatedFileStatus f = itr.next();
+
+ if (!f.isDirectory() && f.getPath().getName().endsWith("jar")) {
+ logger.info("Loading Jar : " + f.getPath().getName());
+ DistributedCache.addFileToClassPath(f.getPath(), config);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ logger.error(e.toString());
+ }
+ }
+
+ public static void addFiletoCache(String libPathStr, Configuration config) {
+
+ try {
+ Path filePath = new Path(libPathStr);
+ DistributedCache.addCacheFile(filePath.toUri(), config);
+ // DistributedCache.createSymlink(config);
+
+ // config.set("mapred.cache.files", libPathStr);
+ // config.set("mapred.create.symlink", "yes");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static Path[] getFileFromCache(String libPathStr,
+ Configuration config) {
+ Path[] localFiles = null;
+ try {
+ logger.info("Local Cache => " + DistributedCache.getLocalCacheFiles(config));
+ logger.info("Hadoop Cache => "+ DistributedCache.getCacheFiles(config));
+ if (DistributedCache.getLocalCacheFiles(config) != null) {
+ localFiles = DistributedCache.getLocalCacheFiles(config);
+ }
+ logger.info("LocalFiles => " + localFiles);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return localFiles;
+ }
+} \ No newline at end of file