blob: af5bdf4b67ca6d2558f23192b2138bf2c322b5a7 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
package parallelai.spyglass.base;
import org.apache.log4j.Logger;
import org.apache.log4j.LogManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
public class JobLibLoader {
private static Logger logger = LogManager.getLogger(JobLibLoader.class);
public static void loadJars(String libPathStr, Configuration config) {
try {
Path libPath = new Path(libPathStr);
FileSystem fs = FileSystem.get(config);
RemoteIterator<LocatedFileStatus> itr = fs.listFiles(libPath, true);
while (itr.hasNext()) {
LocatedFileStatus f = itr.next();
if (!f.isDirectory() && f.getPath().getName().endsWith("jar")) {
logger.info("Loading Jar : " + f.getPath().getName());
DistributedCache.addFileToClassPath(f.getPath(), config);
}
}
} catch (Exception e) {
e.printStackTrace();
logger.error(e.toString());
}
}
public static void addFiletoCache(String libPathStr, Configuration config) {
try {
Path filePath = new Path(libPathStr);
DistributedCache.addCacheFile(filePath.toUri(), config);
// DistributedCache.createSymlink(config);
// config.set("mapred.cache.files", libPathStr);
// config.set("mapred.create.symlink", "yes");
} catch (Exception e) {
e.printStackTrace();
}
}
public static Path[] getFileFromCache(String libPathStr,
Configuration config) {
Path[] localFiles = null;
try {
logger.info("Local Cache => " + DistributedCache.getLocalCacheFiles(config));
logger.info("Hadoop Cache => "+ DistributedCache.getCacheFiles(config));
if (DistributedCache.getLocalCacheFiles(config) != null) {
localFiles = DistributedCache.getLocalCacheFiles(config);
}
logger.info("LocalFiles => " + localFiles);
} catch (Exception e) {
e.printStackTrace();
}
return localFiles;
}
}
|
