dubey/WebGoat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge change from Nanne on testing/loading plugins

Browse Source 
* nbaars-master:
  Fixed loading plugins: sometimes failed file was not correctly extracted
This commit is contained in:
Doug Morato 2015-09-17 22:47:10 -04:00
commit 3f43f56c90
8 changed files with 105 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
webgoat-container/src/main/java/org/owasp/webgoat/plugins/Plugin.java
View File

@ -1,15 +1,13 @@
package org.owasp.webgoat.plugins; package org.owasp.webgoat.plugins;
import com.google.common.base.Optional; import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.owasp.webgoat.classloader.PluginClassLoader; import org.owasp.webgoat.classloader.PluginClassLoader;
import org.owasp.webgoat.lessons.AbstractLesson; import org.owasp.webgoat.lessons.AbstractLesson;
import org.owasp.webgoat.util.LabelProvider; import org.springframework.util.StringUtils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
@ -29,7 +27,6 @@ public class Plugin {
private static final String NAME_LESSON_SOLUTION_DIRECTORY = "lessonSolutions"; private static final String NAME_LESSON_SOLUTION_DIRECTORY = "lessonSolutions";
private static final String NAME_LESSON_PLANS_DIRECTORY = "lessonPlans"; private static final String NAME_LESSON_PLANS_DIRECTORY = "lessonPlans";
private final Path pluginDirectory;
private Class<AbstractLesson> lesson; private Class<AbstractLesson> lesson;
private Map<String, File> solutionLanguageFiles = new HashMap<>(); private Map<String, File> solutionLanguageFiles = new HashMap<>();
@ -37,36 +34,14 @@ public class Plugin {
private List<File> pluginFiles = Lists.newArrayList(); private List<File> pluginFiles = Lists.newArrayList();
private File lessonSourceFile; private File lessonSourceFile;
/** public void findLesson(List<String> classes) {
* <p>Constructor for Plugin.</p>
*
* @param pluginDirectory a {@link java.nio.file.Path} object.
*/
public Plugin(Path pluginDirectory) {
Preconditions.checkNotNull(pluginDirectory, "plugin directory cannot be null");
Preconditions.checkArgument(Files.exists(pluginDirectory), "directory %s does not exists", pluginDirectory);
this.pluginDirectory = pluginDirectory;
}
/**
* <p>Constructor for Plugin.</p>
*
* @param pluginDirectory a {@link java.nio.file.Path} object.
* @param classes a {@link java.util.List} object.
*/
public Plugin(Path pluginDirectory, List<String> classes) {
this(pluginDirectory);
findLesson(classes);
}
private void findLesson(List<String> classes) {
for (String clazzName : classes) { for (String clazzName : classes) {
findLesson(clazzName); findLesson(clazzName);
} }
} }
private void findLesson(String name) { private void findLesson(String name) {
String realClassName = name.replaceFirst("/", "").replaceAll("/", ".").replaceAll(".class", ""); String realClassName = StringUtils.trimLeadingCharacter(name, '/').replaceAll("/", ".").replaceAll(".class", "");
PluginClassLoader cl = (PluginClassLoader) Thread.currentThread().getContextClassLoader(); PluginClassLoader cl = (PluginClassLoader) Thread.currentThread().getContextClassLoader();
try { try {
@ -76,44 +51,23 @@ public class Plugin {
this.lesson = clazz; this.lesson = clazz;
} }
} catch (ClassNotFoundException ce) { } catch (ClassNotFoundException ce) {
throw new PluginLoadingFailure("Class " + realClassName + " listed in jar but unable to load the class.", throw new PluginLoadingFailure("Class " + realClassName + " listed in jar but unable to load the class.", ce);
ce);
} }
} }
/** public void loadFiles(Path file, boolean reload) {
* <p>loadProperties.</p> if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_SOLUTION_DIRECTORY)) {
* solutionLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
* @param properties a {@link java.util.List} object. }
*/ if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_PLANS_DIRECTORY)) {
public void loadProperties(List<Path> properties) { lessonPlansLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
for (Path propertyFile : properties) { }
LabelProvider.updatePluginResources(propertyFile); if (fileEndsWith(file, ".java")) {
LabelProvider.refresh(); lessonSourceFile = file.toFile();
} }
}
/** if (fileEndsWith(file, ".css", ".jsp", ".js")) {
* <p>loadFiles.</p> pluginFiles.add(file.toFile());
*
* @param files a {@link java.util.List} object.
* @param reload a boolean.
*/
public void loadFiles(List<Path> files, boolean reload) {
for (Path file : files) {
if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_SOLUTION_DIRECTORY)) {
solutionLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
}
if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_PLANS_DIRECTORY)) {
lessonPlansLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
}
if (fileEndsWith(file, ".java")) {
lessonSourceFile = file.toFile();
}
if (fileEndsWith(file, ".css", ".jsp", ".js")) {
pluginFiles.add(file.toFile());
}
} }
} }
@ -148,8 +102,6 @@ public class Plugin {
.format("%s/plugin/%s/images/", pluginTarget.getFileName().toString(), this.lesson.getSimpleName()); .format("%s/plugin/%s/images/", pluginTarget.getFileName().toString(), this.lesson.getSimpleName());
replaceInFiles(s, r, pluginFiles); replaceInFiles(s, r, pluginFiles);
replaceInFiles(s, r, Arrays.asList(lessonSourceFile)); replaceInFiles(s, r, Arrays.asList(lessonSourceFile));
} catch (IOException e) { } catch (IOException e) {
throw new PluginLoadingFailure("Unable to rewrite the paths in the solutions", e); throw new PluginLoadingFailure("Unable to rewrite the paths in the solutions", e);
} }
@ -208,4 +160,5 @@ public class Plugin {
return this.lessonPlansLanguageFiles; return this.lessonPlansLanguageFiles;
} }
} }

2
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginBackgroundLoader.java
View File

@ -25,7 +25,7 @@ public class PluginBackgroundLoader implements ServletContextListener {
String targetPath = event.getServletContext().getRealPath("plugin_extracted"); String targetPath = event.getServletContext().getRealPath("plugin_extracted");
scheduler = Executors.newSingleThreadScheduledExecutor(); scheduler = Executors.newSingleThreadScheduledExecutor();
scheduler.scheduleAtFixedRate(new PluginsLoader(Paths.get(pluginPath), Paths.get(targetPath)), 0, 5, TimeUnit.MINUTES); scheduler.scheduleAtFixedRate(new PluginsLoader(Paths.get(pluginPath), Paths.get(targetPath)), 10, 5, TimeUnit.MINUTES);
} }
/** {@inheritDoc} */ /** {@inheritDoc} */

125
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginExtractor.java
View File

@ -1,26 +1,18 @@
package org.owasp.webgoat.plugins; package org.owasp.webgoat.plugins;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.io.Files;
import org.apache.commons.fileupload.util.Streams;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.net.URI;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.Enumeration;
import java.util.List; import java.util.List;
import java.util.zip.ZipEntry;
import static java.lang.String.format; import java.util.zip.ZipFile;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static org.owasp.webgoat.plugins.PluginFileUtils.createDirsIfNotExists;
import static org.owasp.webgoat.plugins.PluginFileUtils.fileEndsWith;
import static org.owasp.webgoat.plugins.PluginFileUtils.hasParentDirectoryWithName;
/** /**
* Extract the jar file and place them in the system temp directory in the folder webgoat and collect the files * Extract the jar file and place them in the system temp directory in the folder webgoat and collect the files
@ -30,49 +22,76 @@ import static org.owasp.webgoat.plugins.PluginFileUtils.hasParentDirectoryWithNa
*/ */
public class PluginExtractor { public class PluginExtractor {
private static final String NAME_LESSON_I18N_DIRECTORY = "i18n";
private final Path pluginArchive;
private final List<String> classes = Lists.newArrayList(); private final List<String> classes = Lists.newArrayList();
private final List<Path> files = new ArrayList<>(); private final List<Path> files = new ArrayList<>();
private final List<Path> properties = new ArrayList<>();
/** public Plugin extractJarFile(final File archive, final File targetDirectory) throws IOException {
* <p>Constructor for PluginExtractor.</p> ZipFile zipFile = new ZipFile(archive);
* Plugin plugin = new Plugin();
* @param pluginArchive a {@link java.nio.file.Path} object. try {
*/ Enumeration<? extends ZipEntry> entries = zipFile.entries();
public PluginExtractor(Path pluginArchive) { while (entries.hasMoreElements()) {
this.pluginArchive = pluginArchive; final ZipEntry zipEntry = entries.nextElement();
} if (shouldProcessFile(zipEntry)) {
boolean processed = processClassFile(zipEntry);
/** if (!processed) {
* <p>extract.</p>
*
* @param target a {@link java.nio.file.Path} object. * @param target a {@link java.nio.file.Path} object.
*/ */
public void extract(final Path target) { public void extract(final Path target) {
try (FileSystem zip = createZipFileSystem()) { try (FileSystem zip = createZipFileSystem()) {
final Path root = zip.getPath("/"); processed = processPropertyFile(zipFile, zipEntry, targetDirectory);
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (file.toString().endsWith(".class")) {
classes.add(file.toString());
} }
if (fileEndsWith(file, ".properties") && hasParentDirectoryWithName(file, if (!processed) {
NAME_LESSON_I18N_DIRECTORY)) { processFile(plugin, zipFile, zipEntry, targetDirectory);
properties.add(Files
.copy(file, createDirsIfNotExists(Paths.get(target.toString(), file.toString())),
REPLACE_EXISTING));
} }
files.add(Files.copy(file, createDirsIfNotExists(Paths.get(target.toString(), file.toString())),
REPLACE_EXISTING));
return FileVisitResult.CONTINUE;
} }
}); }
} catch (Exception e) { } finally {
new PluginLoadingFailure(format("Unable to extract: %s", pluginArchive.getFileName()), e); plugin.findLesson(this.classes);
if (plugin.getLesson().isPresent()) {
plugin.rewritePaths(targetDirectory.toPath());
}
zipFile.close();
} }
return plugin;
}
private void processFile(Plugin plugin, ZipFile zipFile, ZipEntry zipEntry, File targetDirectory)
throws IOException {
final File targetFile = new File(targetDirectory, zipEntry.getName());
copyFile(zipFile, zipEntry, targetFile, false);
plugin.loadFiles(targetFile.toPath(), true);
}
private boolean processPropertyFile(ZipFile zipFile, ZipEntry zipEntry, File targetDirectory)
throws IOException {
if (zipEntry.getName().endsWith(".properties")) {
final File targetFile = new File(targetDirectory, zipEntry.getName());
copyFile(zipFile, zipEntry, targetFile, true);
return true;
}
return false;
}
private boolean processClassFile(ZipEntry zipEntry) {
if (zipEntry.getName().endsWith(".class")) {
classes.add(zipEntry.getName());
return true;
}
return false;
}
private boolean shouldProcessFile(ZipEntry zipEntry) {
return !zipEntry.isDirectory() && !zipEntry.getName().startsWith("META-INF");
}
private File copyFile(ZipFile zipFile, ZipEntry zipEntry, File targetFile, boolean append) throws IOException {
Files.createParentDirs(targetFile);
try (FileOutputStream fos = new FileOutputStream(targetFile, append)) {
Streams.copy(zipFile.getInputStream(zipEntry), fos, true);
}
return targetFile;
} }
/** /**
@ -92,18 +111,4 @@ public class PluginExtractor {
public List<Path> getFiles() { public List<Path> getFiles() {
return this.files; return this.files;
} }
/**
* <p>Getter for the field <code>properties</code>.</p>
*
* @return a {@link java.util.List} object.
*/
public List<Path> getProperties() {
return this.properties;
}
private FileSystem createZipFileSystem() throws Exception {
final URI uri = URI.create("jar:file:" + pluginArchive.toUri().getPath());
return FileSystems.newFileSystem(uri, new HashMap<String, Object>());
}
} }

55
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginFileUtils.java
View File

@ -2,17 +2,15 @@ package org.owasp.webgoat.plugins;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.commons.io.IOUtils;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.file.DirectoryStream; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List;
/** /**
* <p>PluginFileUtils class.</p> * <p>PluginFileUtils class.</p>
@ -79,24 +77,6 @@ public class PluginFileUtils {
return p; return p;
} }
/**
* <p>getFilesInDirectory.</p>
*
* @param directory a {@link java.nio.file.Path} object.
* @return a {@link java.util.List} object.
* @throws java.io.IOException if any.
*/
public static List<Path> getFilesInDirectory(Path directory) throws IOException {
List<Path> files = new ArrayList<>();
DirectoryStream<Path> dirStream;
dirStream = Files.newDirectoryStream(directory);
for (Path entry : dirStream) {
files.add(entry);
}
dirStream.close();
return files;
}
/** /**
* <p>replaceInFiles.</p> * <p>replaceInFiles.</p>
* *
@ -111,7 +91,7 @@ public class PluginFileUtils {
Preconditions.checkNotNull(files); Preconditions.checkNotNull(files);
for (File file : files) { for (File file : files) {
replaceInFile(replace, with, Paths.get(file.toURI())); replaceInFile(replace, with, file);
} }
} }
@ -123,31 +103,16 @@ public class PluginFileUtils {
* @param file a {@link java.nio.file.Path} object. * @param file a {@link java.nio.file.Path} object.
* @throws java.io.IOException if any. * @throws java.io.IOException if any.
*/ */
public static void replaceInFile(String replace, String with, Path file) throws IOException { public static void replaceInFile(String replace, String with, File file) throws IOException {
Preconditions.checkNotNull(replace); Preconditions.checkNotNull(replace);
Preconditions.checkNotNull(with); Preconditions.checkNotNull(with);
Preconditions.checkNotNull(file); Preconditions.checkNotNull(file);
byte[] fileAsBytes = Files.readAllBytes(file); String fileAsString = "";
String fileAsString = new String(fileAsBytes); try (FileInputStream fis = new FileInputStream(file);) {
fileAsString = fileAsString.replaceAll(replace, with); fileAsString = IOUtils.toString(fis, StandardCharsets.UTF_8.name());
Files.write(file, fileAsString.getBytes()); fileAsString = fileAsString.replaceAll(replace, with);
}
/**
* <p>writeFile.</p>
*
* @param targetFile a {@link java.nio.file.Path} object.
* @param bytes an array of byte.
* @param options a {@link java.nio.file.OpenOption} object.
* @throws java.io.IOException if any.
*/
public static void writeFile(Path targetFile, byte[] bytes, OpenOption... options) throws IOException {
createDirsIfNotExists(targetFile.getParent());
if (!Files.exists(targetFile)) {
Files.createFile(targetFile);
} }
Files.write(targetFile, bytes, options); Files.write(file.toPath(), fileAsString.getBytes());
} }
} }

32
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginsLoader.java
View File

@ -6,13 +6,13 @@ import org.owasp.webgoat.classloader.PluginClassLoader;
import org.owasp.webgoat.util.LabelProvider; import org.owasp.webgoat.util.LabelProvider;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.util.ResourceUtils;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.file.FileVisitResult; import java.nio.file.FileVisitResult;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor; import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
import java.util.List; import java.util.List;
@ -88,39 +88,33 @@ public class PluginsLoader implements Runnable {
private List<Plugin> processPlugins(List<URL> jars, boolean reload) throws Exception { private List<Plugin> processPlugins(List<URL> jars, boolean reload) throws Exception {
final List<Plugin> plugins = Lists.newArrayList(); final List<Plugin> plugins = Lists.newArrayList();
final ExecutorService executorService = Executors.newFixedThreadPool(20); final ExecutorService executorService = Executors.newFixedThreadPool(10);
final CompletionService<PluginExtractor> completionService = new ExecutorCompletionService<>(executorService); final CompletionService<Plugin> completionService = new ExecutorCompletionService<>(executorService);
final List<Callable<PluginExtractor>> callables = extractJars(jars); final List<Callable<Plugin>> callables = extractJars(jars);
for (Callable<PluginExtractor> s : callables) { for (Callable<Plugin> s : callables) {
completionService.submit(s); completionService.submit(s);
} }
int n = callables.size(); int n = callables.size();
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
PluginExtractor extractor = completionService.take().get(); Plugin plugin = completionService.take().get();
Plugin plugin = new Plugin(pluginTarget, extractor.getClasses());
if (plugin.getLesson().isPresent()) { if (plugin.getLesson().isPresent()) {
PluginFileUtils.createDirsIfNotExists(pluginTarget);
plugin.loadFiles(extractor.getFiles(), reload);
plugin.loadProperties(extractor.getProperties());
plugin.rewritePaths(pluginTarget);
plugins.add(plugin); plugins.add(plugin);
} }
} }
LabelProvider.refresh(); LabelProvider.updatePluginResources(pluginTarget.resolve("plugin/i18n/WebGoatLabels.properties"));
return plugins; return plugins;
} }
private List<Callable<PluginExtractor>> extractJars(List<URL> jars) { private List<Callable<Plugin>> extractJars(List<URL> jars) {
List<Callable<PluginExtractor>> extractorCallables = Lists.newArrayList(); List<Callable<Plugin>> extractorCallables = Lists.newArrayList();
for (final URL jar : jars) { for (final URL jar : jars) {
extractorCallables.add(new Callable<PluginExtractor>() { extractorCallables.add(new Callable<Plugin>() {
@Override @Override
public PluginExtractor call() throws Exception { public Plugin call() throws Exception {
PluginExtractor extractor = new PluginExtractor(Paths.get(jar.toURI())); PluginExtractor extractor = new PluginExtractor();
extractor.extract(pluginTarget); return extractor.extractJarFile(ResourceUtils.getFile(jar), pluginTarget.toFile());
return extractor;
} }
}); });
} }

5
webgoat-container/src/main/java/org/owasp/webgoat/util/LabelProvider.java
View File

@ -6,7 +6,6 @@ import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader; import org.springframework.core.io.ResourceLoader;
import org.springframework.core.io.UrlResource; import org.springframework.core.io.UrlResource;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.DefaultPropertiesPersister;
import javax.inject.Singleton; import javax.inject.Singleton;
import java.net.MalformedURLException; import java.net.MalformedURLException;
@ -67,9 +66,6 @@ public class LabelProvider {
labels.setFallbackToSystemLocale(false); labels.setFallbackToSystemLocale(false);
labels.setUseCodeAsDefaultMessage(true); labels.setUseCodeAsDefaultMessage(true);
pluginLabels.setParentMessageSource(labels); pluginLabels.setParentMessageSource(labels);
pluginLabels.setPropertiesPersister(new DefaultPropertiesPersister() {
});
} }
/** /**
@ -96,7 +92,6 @@ public class LabelProvider {
return Thread.currentThread().getContextClassLoader(); return Thread.currentThread().getContextClassLoader();
} }
}); });
}
/** /**
* <p>refresh.</p> * <p>refresh.</p>

1
webgoat-container/src/test/java/org/owasp/webgoat/plugins/WebGoatIT.java
View File

@ -232,7 +232,6 @@ public class WebGoatIT implements SauceOnDemandSessionIdProvider {
@Test @Test
public void testStartMvc() { public void testStartMvc() {
driver.get(baseWebGoatUrl + "/start.mvc"); driver.get(baseWebGoatUrl + "/start.mvc");
WebDriverWait wait = new WebDriverWait(driver, 15); // wait for a maximum of 15 seconds WebDriverWait wait = new WebDriverWait(driver, 15); // wait for a maximum of 15 seconds

1
webgoat-container/src/test/java/org/owasp/webgoat/util/LabelProviderTest.java
View File

@ -22,7 +22,6 @@ public class LabelProviderTest {
public void loadingPluginLabels() throws IOException { public void loadingPluginLabels() throws IOException {
LabelProvider labelProvider = new LabelProvider(); LabelProvider labelProvider = new LabelProvider();
labelProvider.updatePluginResources(new ClassPathResource("log4j.properties").getFile().toPath()); labelProvider.updatePluginResources(new ClassPathResource("log4j.properties").getFile().toPath());
LabelProvider.refresh();
assertThat(labelProvider.get(Locale.ENGLISH, "LessonCompleted"), CoreMatchers.equalTo( assertThat(labelProvider.get(Locale.ENGLISH, "LessonCompleted"), CoreMatchers.equalTo(
"Congratulations. You have successfully completed this lesson.")); "Congratulations. You have successfully completed this lesson."));
assertThat(labelProvider.get(Locale.ENGLISH, "log4j.appender.CONSOLE.Target"), CoreMatchers.equalTo( assertThat(labelProvider.get(Locale.ENGLISH, "log4j.appender.CONSOLE.Target"), CoreMatchers.equalTo(