aboutsummaryrefslogtreecommitdiffstats
path: root/src/main/java/parallelai/spyglass/base/JobLibLoader.java
blob: f49cca9c0819214263ac1ca6d16f0a4b77e03bdc (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
package parallelai.spyglass.base;

import org.apache.log4j.Logger;
import org.apache.log4j.LogManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;

public class JobLibLoader {

	private static Logger logger = LogManager.getLogger(JobLibLoader.class);
	
	public static void loadJars(String libPathStr, Configuration config) {
		
		try {
			Path libPath = new Path(libPathStr);

			FileSystem fs = FileSystem.get(config);

			RemoteIterator<LocatedFileStatus> itr = fs.listFiles(libPath, true);

			while (itr.hasNext()) {
				LocatedFileStatus f = itr.next();

				if (!f.isDirectory() && f.getPath().getName().endsWith("jar")) {
					logger.info("Loading Jar : " + f.getPath().getName());
					DistributedCache.addFileToClassPath(f.getPath(), config);
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
			logger.error(e.toString());
		}
	}

	public static void addFiletoCache(String libPathStr, Configuration config) {

		try {
			Path filePath = new Path(libPathStr);
			DistributedCache.addCacheFile(filePath.toUri(), config);
			// DistributedCache.createSymlink(config);

			// config.set("mapred.cache.files", libPathStr);
			// config.set("mapred.create.symlink", "yes");

		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	public static Path[] getFileFromCache(String libPathStr,
			Configuration config) {
		Path[] localFiles = null;
		try {
			logger.info("Local Cache => " + DistributedCache.getLocalCacheFiles(config));
			logger.info("Hadoop Cache => "+ DistributedCache.getCacheFiles(config));
			if (DistributedCache.getLocalCacheFiles(config) != null) {
				localFiles = DistributedCache.getLocalCacheFiles(config);
			}
			logger.info("LocalFiles => " + localFiles);
		} catch (Exception e) {
			e.printStackTrace();
		}
		return localFiles;
	}
}