dubey/WebGoat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed loading plugins: sometimes failed file was not correctly extracted

Browse Source
This commit is contained in:
Nanne Baars 2015-09-17 07:29:51 +02:00
parent d142407de3
commit 9474f66d96
8 changed files with 107 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
webgoat-container/src/main/java/org/owasp/webgoat/plugins/Plugin.java
View File

@ -1,15 +1,13 @@
package org.owasp.webgoat.plugins; package org.owasp.webgoat.plugins;
import com.google.common.base.Optional; import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.owasp.webgoat.classloader.PluginClassLoader; import org.owasp.webgoat.classloader.PluginClassLoader;
import org.owasp.webgoat.lessons.AbstractLesson; import org.owasp.webgoat.lessons.AbstractLesson;
import org.owasp.webgoat.util.LabelProvider; import org.springframework.util.StringUtils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
@ -24,7 +22,6 @@ public class Plugin {
private static final String NAME_LESSON_SOLUTION_DIRECTORY = "lessonSolutions"; private static final String NAME_LESSON_SOLUTION_DIRECTORY = "lessonSolutions";
private static final String NAME_LESSON_PLANS_DIRECTORY = "lessonPlans"; private static final String NAME_LESSON_PLANS_DIRECTORY = "lessonPlans";
private final Path pluginDirectory;
private Class<AbstractLesson> lesson; private Class<AbstractLesson> lesson;
private Map<String, File> solutionLanguageFiles = new HashMap<>(); private Map<String, File> solutionLanguageFiles = new HashMap<>();
@ -32,25 +29,14 @@ public class Plugin {
private List<File> pluginFiles = Lists.newArrayList(); private List<File> pluginFiles = Lists.newArrayList();
private File lessonSourceFile; private File lessonSourceFile;
public Plugin(Path pluginDirectory) { public void findLesson(List<String> classes) {
Preconditions.checkNotNull(pluginDirectory, "plugin directory cannot be null");
Preconditions.checkArgument(Files.exists(pluginDirectory), "directory %s does not exists", pluginDirectory);
this.pluginDirectory = pluginDirectory;
}
public Plugin(Path pluginDirectory, List<String> classes) {
this(pluginDirectory);
findLesson(classes);
}
private void findLesson(List<String> classes) {
for (String clazzName : classes) { for (String clazzName : classes) {
findLesson(clazzName); findLesson(clazzName);
} }
} }
private void findLesson(String name) { private void findLesson(String name) {
String realClassName = name.replaceFirst("/", "").replaceAll("/", ".").replaceAll(".class", ""); String realClassName = StringUtils.trimLeadingCharacter(name, '/').replaceAll("/", ".").replaceAll(".class", "");
PluginClassLoader cl = (PluginClassLoader) Thread.currentThread().getContextClassLoader(); PluginClassLoader cl = (PluginClassLoader) Thread.currentThread().getContextClassLoader();
try { try {
@ -60,33 +46,23 @@ public class Plugin {
this.lesson = clazz; this.lesson = clazz;
} }
} catch (ClassNotFoundException ce) { } catch (ClassNotFoundException ce) {
throw new PluginLoadingFailure("Class " + realClassName + " listed in jar but unable to load the class.", throw new PluginLoadingFailure("Class " + realClassName + " listed in jar but unable to load the class.", ce);
ce);
} }
} }
public void loadProperties(List<Path> properties) { public void loadFiles(Path file, boolean reload) {
for (Path propertyFile : properties) { if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_SOLUTION_DIRECTORY)) {
LabelProvider.updatePluginResources(propertyFile); solutionLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
LabelProvider.refresh(); }
if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_PLANS_DIRECTORY)) {
lessonPlansLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
}
if (fileEndsWith(file, ".java")) {
lessonSourceFile = file.toFile();
} }
}
public void loadFiles(List<Path> files, boolean reload) { if (fileEndsWith(file, ".css", ".jsp", ".js")) {
for (Path file : files) { pluginFiles.add(file.toFile());
if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_SOLUTION_DIRECTORY)) {
solutionLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
}
if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_PLANS_DIRECTORY)) {
lessonPlansLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
}
if (fileEndsWith(file, ".java")) {
lessonSourceFile = file.toFile();
}
if (fileEndsWith(file, ".css", ".jsp", ".js")) {
pluginFiles.add(file.toFile());
}
} }
} }
@ -116,8 +92,6 @@ public class Plugin {
.format("%s/plugin/%s/images/", pluginTarget.getFileName().toString(), this.lesson.getSimpleName()); .format("%s/plugin/%s/images/", pluginTarget.getFileName().toString(), this.lesson.getSimpleName());
replaceInFiles(s, r, pluginFiles); replaceInFiles(s, r, pluginFiles);
replaceInFiles(s, r, Arrays.asList(lessonSourceFile)); replaceInFiles(s, r, Arrays.asList(lessonSourceFile));
} catch (IOException e) { } catch (IOException e) {
throw new PluginLoadingFailure("Unable to rewrite the paths in the solutions", e); throw new PluginLoadingFailure("Unable to rewrite the paths in the solutions", e);
} }
@ -153,4 +127,5 @@ public class Plugin {
return this.lessonPlansLanguageFiles; return this.lessonPlansLanguageFiles;
} }
} }

2
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginBackgroundLoader.java
View File

@ -19,7 +19,7 @@ public class PluginBackgroundLoader implements ServletContextListener {
String targetPath = event.getServletContext().getRealPath("plugin_extracted"); String targetPath = event.getServletContext().getRealPath("plugin_extracted");
scheduler = Executors.newSingleThreadScheduledExecutor(); scheduler = Executors.newSingleThreadScheduledExecutor();
scheduler.scheduleAtFixedRate(new PluginsLoader(Paths.get(pluginPath), Paths.get(targetPath)), 0, 5, TimeUnit.MINUTES); scheduler.scheduleAtFixedRate(new PluginsLoader(Paths.get(pluginPath), Paths.get(targetPath)), 10, 5, TimeUnit.MINUTES);
} }
@Override @Override

118
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginExtractor.java
View File

@ -1,26 +1,18 @@
package org.owasp.webgoat.plugins; package org.owasp.webgoat.plugins;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.io.Files;
import org.apache.commons.fileupload.util.Streams;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.net.URI;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.Enumeration;
import java.util.List; import java.util.List;
import java.util.zip.ZipEntry;
import static java.lang.String.format; import java.util.zip.ZipFile;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static org.owasp.webgoat.plugins.PluginFileUtils.createDirsIfNotExists;
import static org.owasp.webgoat.plugins.PluginFileUtils.fileEndsWith;
import static org.owasp.webgoat.plugins.PluginFileUtils.hasParentDirectoryWithName;
/** /**
* Extract the jar file and place them in the system temp directory in the folder webgoat and collect the files * Extract the jar file and place them in the system temp directory in the folder webgoat and collect the files
@ -28,39 +20,72 @@ import static org.owasp.webgoat.plugins.PluginFileUtils.hasParentDirectoryWithNa
*/ */
public class PluginExtractor { public class PluginExtractor {
private static final String NAME_LESSON_I18N_DIRECTORY = "i18n";
private final Path pluginArchive;
private final List<String> classes = Lists.newArrayList(); private final List<String> classes = Lists.newArrayList();
private final List<Path> files = new ArrayList<>(); private final List<Path> files = new ArrayList<>();
private final List<Path> properties = new ArrayList<>();
public PluginExtractor(Path pluginArchive) { public Plugin extractJarFile(final File archive, final File targetDirectory) throws IOException {
this.pluginArchive = pluginArchive; ZipFile zipFile = new ZipFile(archive);
Plugin plugin = new Plugin();
try {
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
final ZipEntry zipEntry = entries.nextElement();
if (shouldProcessFile(zipEntry)) {
boolean processed = processClassFile(zipEntry);
if (!processed) {
processed = processPropertyFile(zipFile, zipEntry, targetDirectory);
}
if (!processed) {
processFile(plugin, zipFile, zipEntry, targetDirectory);
}
}
}
} finally {
plugin.findLesson(this.classes);
if (plugin.getLesson().isPresent()) {
plugin.rewritePaths(targetDirectory.toPath());
}
zipFile.close();
}
return plugin;
} }
public void extract(final Path target) { private void processFile(Plugin plugin, ZipFile zipFile, ZipEntry zipEntry, File targetDirectory)
try (FileSystem zip = createZipFileSystem()) { throws IOException {
final Path root = zip.getPath("/"); final File targetFile = new File(targetDirectory, zipEntry.getName());
Files.walkFileTree(root, new SimpleFileVisitor<Path>() { copyFile(zipFile, zipEntry, targetFile, false);
@Override plugin.loadFiles(targetFile.toPath(), true);
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { }
if (file.toString().endsWith(".class")) {
classes.add(file.toString()); private boolean processPropertyFile(ZipFile zipFile, ZipEntry zipEntry, File targetDirectory)
} throws IOException {
if (fileEndsWith(file, ".properties") && hasParentDirectoryWithName(file, if (zipEntry.getName().endsWith(".properties")) {
NAME_LESSON_I18N_DIRECTORY)) { final File targetFile = new File(targetDirectory, zipEntry.getName());
properties.add(Files copyFile(zipFile, zipEntry, targetFile, true);
.copy(file, createDirsIfNotExists(Paths.get(target.toString(), file.toString())), return true;
REPLACE_EXISTING));
}
files.add(Files.copy(file, createDirsIfNotExists(Paths.get(target.toString(), file.toString())),
REPLACE_EXISTING));
return FileVisitResult.CONTINUE;
}
});
} catch (Exception e) {
new PluginLoadingFailure(format("Unable to extract: %s", pluginArchive.getFileName()), e);
} }
return false;
}
private boolean processClassFile(ZipEntry zipEntry) {
if (zipEntry.getName().endsWith(".class")) {
classes.add(zipEntry.getName());
return true;
}
return false;
}
private boolean shouldProcessFile(ZipEntry zipEntry) {
return !zipEntry.isDirectory() && !zipEntry.getName().startsWith("META-INF");
}
private File copyFile(ZipFile zipFile, ZipEntry zipEntry, File targetFile, boolean append) throws IOException {
Files.createParentDirs(targetFile);
try (FileOutputStream fos = new FileOutputStream(targetFile, append)) {
Streams.copy(zipFile.getInputStream(zipEntry), fos, true);
}
return targetFile;
} }
public List<String> getClasses() { public List<String> getClasses() {
@ -70,13 +95,4 @@ public class PluginExtractor {
public List<Path> getFiles() { public List<Path> getFiles() {
return this.files; return this.files;
} }
public List<Path> getProperties() {
return this.properties;
}
private FileSystem createZipFileSystem() throws Exception {
final URI uri = URI.create("jar:file:" + pluginArchive.toUri().getPath());
return FileSystems.newFileSystem(uri, new HashMap<String, Object>());
}
} }

40
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginFileUtils.java
View File

@ -2,17 +2,15 @@ package org.owasp.webgoat.plugins;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.commons.io.IOUtils;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.file.DirectoryStream; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List;
public class PluginFileUtils { public class PluginFileUtils {
@ -46,44 +44,26 @@ public class PluginFileUtils {
return p; return p;
} }
public static List<Path> getFilesInDirectory(Path directory) throws IOException {
List<Path> files = new ArrayList<>();
DirectoryStream<Path> dirStream;
dirStream = Files.newDirectoryStream(directory);
for (Path entry : dirStream) {
files.add(entry);
}
dirStream.close();
return files;
}
public static void replaceInFiles(String replace, String with, Collection<File> files) throws IOException { public static void replaceInFiles(String replace, String with, Collection<File> files) throws IOException {
Preconditions.checkNotNull(replace); Preconditions.checkNotNull(replace);
Preconditions.checkNotNull(with); Preconditions.checkNotNull(with);
Preconditions.checkNotNull(files); Preconditions.checkNotNull(files);
for (File file : files) { for (File file : files) {
replaceInFile(replace, with, Paths.get(file.toURI())); replaceInFile(replace, with, file);
} }
} }
public static void replaceInFile(String replace, String with, Path file) throws IOException { public static void replaceInFile(String replace, String with, File file) throws IOException {
Preconditions.checkNotNull(replace); Preconditions.checkNotNull(replace);
Preconditions.checkNotNull(with); Preconditions.checkNotNull(with);
Preconditions.checkNotNull(file); Preconditions.checkNotNull(file);
byte[] fileAsBytes = Files.readAllBytes(file); String fileAsString = "";
String fileAsString = new String(fileAsBytes); try (FileInputStream fis = new FileInputStream(file);) {
fileAsString = fileAsString.replaceAll(replace, with); fileAsString = IOUtils.toString(fis, StandardCharsets.UTF_8.name());
Files.write(file, fileAsString.getBytes()); fileAsString = fileAsString.replaceAll(replace, with);
}
public static void writeFile(Path targetFile, byte[] bytes, OpenOption... options) throws IOException {
createDirsIfNotExists(targetFile.getParent());
if (!Files.exists(targetFile)) {
Files.createFile(targetFile);
} }
Files.write(targetFile, bytes, options); Files.write(file.toPath(), fileAsString.getBytes());
} }
} }

32
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginsLoader.java
View File

@ -6,13 +6,13 @@ import org.owasp.webgoat.classloader.PluginClassLoader;
import org.owasp.webgoat.util.LabelProvider; import org.owasp.webgoat.util.LabelProvider;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.util.ResourceUtils;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.file.FileVisitResult; import java.nio.file.FileVisitResult;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor; import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
import java.util.List; import java.util.List;
@ -70,39 +70,33 @@ public class PluginsLoader implements Runnable {
private List<Plugin> processPlugins(List<URL> jars, boolean reload) throws Exception { private List<Plugin> processPlugins(List<URL> jars, boolean reload) throws Exception {
final List<Plugin> plugins = Lists.newArrayList(); final List<Plugin> plugins = Lists.newArrayList();
final ExecutorService executorService = Executors.newFixedThreadPool(20); final ExecutorService executorService = Executors.newFixedThreadPool(10);
final CompletionService<PluginExtractor> completionService = new ExecutorCompletionService<>(executorService); final CompletionService<Plugin> completionService = new ExecutorCompletionService<>(executorService);
final List<Callable<PluginExtractor>> callables = extractJars(jars); final List<Callable<Plugin>> callables = extractJars(jars);
for (Callable<PluginExtractor> s : callables) { for (Callable<Plugin> s : callables) {
completionService.submit(s); completionService.submit(s);
} }
int n = callables.size(); int n = callables.size();
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
PluginExtractor extractor = completionService.take().get(); Plugin plugin = completionService.take().get();
Plugin plugin = new Plugin(pluginTarget, extractor.getClasses());
if (plugin.getLesson().isPresent()) { if (plugin.getLesson().isPresent()) {
PluginFileUtils.createDirsIfNotExists(pluginTarget);
plugin.loadFiles(extractor.getFiles(), reload);
plugin.loadProperties(extractor.getProperties());
plugin.rewritePaths(pluginTarget);
plugins.add(plugin); plugins.add(plugin);
} }
} }
LabelProvider.refresh(); LabelProvider.updatePluginResources(pluginTarget.resolve("plugin/i18n/WebGoatLabels.properties"));
return plugins; return plugins;
} }
private List<Callable<PluginExtractor>> extractJars(List<URL> jars) { private List<Callable<Plugin>> extractJars(List<URL> jars) {
List<Callable<PluginExtractor>> extractorCallables = Lists.newArrayList(); List<Callable<Plugin>> extractorCallables = Lists.newArrayList();
for (final URL jar : jars) { for (final URL jar : jars) {
extractorCallables.add(new Callable<PluginExtractor>() { extractorCallables.add(new Callable<Plugin>() {
@Override @Override
public PluginExtractor call() throws Exception { public Plugin call() throws Exception {
PluginExtractor extractor = new PluginExtractor(Paths.get(jar.toURI())); PluginExtractor extractor = new PluginExtractor();
extractor.extract(pluginTarget); return extractor.extractJarFile(ResourceUtils.getFile(jar), pluginTarget.toFile());
return extractor;
} }
}); });
} }

6
webgoat-container/src/main/java/org/owasp/webgoat/util/LabelProvider.java
View File

@ -6,7 +6,6 @@ import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader; import org.springframework.core.io.ResourceLoader;
import org.springframework.core.io.UrlResource; import org.springframework.core.io.UrlResource;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.DefaultPropertiesPersister;
import javax.inject.Singleton; import javax.inject.Singleton;
import java.net.MalformedURLException; import java.net.MalformedURLException;
@ -59,9 +58,6 @@ public class LabelProvider {
labels.setFallbackToSystemLocale(false); labels.setFallbackToSystemLocale(false);
labels.setUseCodeAsDefaultMessage(true); labels.setUseCodeAsDefaultMessage(true);
pluginLabels.setParentMessageSource(labels); pluginLabels.setParentMessageSource(labels);
pluginLabels.setPropertiesPersister(new DefaultPropertiesPersister() {
});
} }
public static void updatePluginResources(final Path propertyFile) { public static void updatePluginResources(final Path propertyFile) {
@ -83,9 +79,7 @@ public class LabelProvider {
return Thread.currentThread().getContextClassLoader(); return Thread.currentThread().getContextClassLoader();
} }
}); });
}
public static void refresh() {
pluginLabels.clearCache(); pluginLabels.clearCache();
} }

1
webgoat-container/src/test/java/org/owasp/webgoat/plugins/WebGoatIT.java
View File

@ -219,7 +219,6 @@ public class WebGoatIT implements SauceOnDemandSessionIdProvider {
@Test @Test
public void testStartMvc() { public void testStartMvc() {
driver.get(baseWebGoatUrl + "/start.mvc"); driver.get(baseWebGoatUrl + "/start.mvc");
WebDriverWait wait = new WebDriverWait(driver, 10); // wait for a maximum of 5 seconds WebDriverWait wait = new WebDriverWait(driver, 10); // wait for a maximum of 5 seconds

1
webgoat-container/src/test/java/org/owasp/webgoat/util/LabelProviderTest.java
View File

@ -22,7 +22,6 @@ public class LabelProviderTest {
public void loadingPluginLabels() throws IOException { public void loadingPluginLabels() throws IOException {
LabelProvider labelProvider = new LabelProvider(); LabelProvider labelProvider = new LabelProvider();
labelProvider.updatePluginResources(new ClassPathResource("log4j.properties").getFile().toPath()); labelProvider.updatePluginResources(new ClassPathResource("log4j.properties").getFile().toPath());
LabelProvider.refresh();
assertThat(labelProvider.get(Locale.ENGLISH, "LessonCompleted"), CoreMatchers.equalTo( assertThat(labelProvider.get(Locale.ENGLISH, "LessonCompleted"), CoreMatchers.equalTo(
"Congratulations. You have successfully completed this lesson.")); "Congratulations. You have successfully completed this lesson."));
assertThat(labelProvider.get(Locale.ENGLISH, "log4j.appender.CONSOLE.Target"), CoreMatchers.equalTo( assertThat(labelProvider.get(Locale.ENGLISH, "log4j.appender.CONSOLE.Target"), CoreMatchers.equalTo(