dubey/WebGoat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge change from Nanne on testing/loading plugins

Browse Source 
* nbaars-master:
  Fixed loading plugins: sometimes failed file was not correctly extracted
This commit is contained in:
Doug Morato 2015-09-17 22:47:10 -04:00
commit 3f43f56c90
8 changed files with 105 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
webgoat-container/src/main/java/org/owasp/webgoat/plugins/Plugin.java
View File

@ -1,15 +1,13 @@
package org.owasp.webgoat.plugins;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.owasp.webgoat.classloader.PluginClassLoader;
import org.owasp.webgoat.lessons.AbstractLesson;
import org.owasp.webgoat.util.LabelProvider;
import org.springframework.util.StringUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashMap;
@ -29,7 +27,6 @@ public class Plugin {
private static final String NAME_LESSON_SOLUTION_DIRECTORY = "lessonSolutions";
private static final String NAME_LESSON_PLANS_DIRECTORY = "lessonPlans";
private final Path pluginDirectory;
private Class<AbstractLesson> lesson;
private Map<String, File> solutionLanguageFiles = new HashMap<>();
@ -37,36 +34,14 @@ public class Plugin {
private List<File> pluginFiles = Lists.newArrayList();
private File lessonSourceFile;
/**
* <p>Constructor for Plugin.</p>
*
* @param pluginDirectory a {@link java.nio.file.Path} object.
*/
public Plugin(Path pluginDirectory) {
Preconditions.checkNotNull(pluginDirectory, "plugin directory cannot be null");
Preconditions.checkArgument(Files.exists(pluginDirectory), "directory %s does not exists", pluginDirectory);
this.pluginDirectory = pluginDirectory;
}
/**
* <p>Constructor for Plugin.</p>
*
* @param pluginDirectory a {@link java.nio.file.Path} object.
* @param classes a {@link java.util.List} object.
*/
public Plugin(Path pluginDirectory, List<String> classes) {
this(pluginDirectory);
findLesson(classes);
}
private void findLesson(List<String> classes) {
public void findLesson(List<String> classes) {
for (String clazzName : classes) {
findLesson(clazzName);
}
}
private void findLesson(String name) {
String realClassName = name.replaceFirst("/", "").replaceAll("/", ".").replaceAll(".class", "");
String realClassName = StringUtils.trimLeadingCharacter(name, '/').replaceAll("/", ".").replaceAll(".class", "");
PluginClassLoader cl = (PluginClassLoader) Thread.currentThread().getContextClassLoader();
try {
@ -76,44 +51,23 @@ public class Plugin {
this.lesson = clazz;
}
} catch (ClassNotFoundException ce) {
throw new PluginLoadingFailure("Class " + realClassName + " listed in jar but unable to load the class.",
ce);
throw new PluginLoadingFailure("Class " + realClassName + " listed in jar but unable to load the class.", ce);
}
}
/**
* <p>loadProperties.</p>
*
* @param properties a {@link java.util.List} object.
*/
public void loadProperties(List<Path> properties) {
for (Path propertyFile : properties) {
LabelProvider.updatePluginResources(propertyFile);
LabelProvider.refresh();
public void loadFiles(Path file, boolean reload) {
if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_SOLUTION_DIRECTORY)) {
solutionLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
}
if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_PLANS_DIRECTORY)) {
lessonPlansLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
}
if (fileEndsWith(file, ".java")) {
lessonSourceFile = file.toFile();
}
}
/**
* <p>loadFiles.</p>
*
* @param files a {@link java.util.List} object.
* @param reload a boolean.
*/
public void loadFiles(List<Path> files, boolean reload) {
for (Path file : files) {
if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_SOLUTION_DIRECTORY)) {
solutionLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
}
if (fileEndsWith(file, ".html") && hasParentDirectoryWithName(file, NAME_LESSON_PLANS_DIRECTORY)) {
lessonPlansLanguageFiles.put(file.getParent().getFileName().toString(), file.toFile());
}
if (fileEndsWith(file, ".java")) {
lessonSourceFile = file.toFile();
}
if (fileEndsWith(file, ".css", ".jsp", ".js")) {
pluginFiles.add(file.toFile());
}
if (fileEndsWith(file, ".css", ".jsp", ".js")) {
pluginFiles.add(file.toFile());
}
}
@ -148,8 +102,6 @@ public class Plugin {
.format("%s/plugin/%s/images/", pluginTarget.getFileName().toString(), this.lesson.getSimpleName());
replaceInFiles(s, r, pluginFiles);
replaceInFiles(s, r, Arrays.asList(lessonSourceFile));
} catch (IOException e) {
throw new PluginLoadingFailure("Unable to rewrite the paths in the solutions", e);
}
@ -208,4 +160,5 @@ public class Plugin {
return this.lessonPlansLanguageFiles;
}
}

2
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginBackgroundLoader.java
View File

@ -25,7 +25,7 @@ public class PluginBackgroundLoader implements ServletContextListener {
String targetPath = event.getServletContext().getRealPath("plugin_extracted");
scheduler = Executors.newSingleThreadScheduledExecutor();
scheduler.scheduleAtFixedRate(new PluginsLoader(Paths.get(pluginPath), Paths.get(targetPath)), 0, 5, TimeUnit.MINUTES);
scheduler.scheduleAtFixedRate(new PluginsLoader(Paths.get(pluginPath), Paths.get(targetPath)), 10, 5, TimeUnit.MINUTES);
}
/** {@inheritDoc} */

125
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginExtractor.java
View File

@ -1,26 +1,18 @@
package org.owasp.webgoat.plugins;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import org.apache.commons.fileupload.util.Streams;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Enumeration;
import java.util.List;
import static java.lang.String.format;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static org.owasp.webgoat.plugins.PluginFileUtils.createDirsIfNotExists;
import static org.owasp.webgoat.plugins.PluginFileUtils.fileEndsWith;
import static org.owasp.webgoat.plugins.PluginFileUtils.hasParentDirectoryWithName;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
/**
* Extract the jar file and place them in the system temp directory in the folder webgoat and collect the files
@ -30,49 +22,76 @@ import static org.owasp.webgoat.plugins.PluginFileUtils.hasParentDirectoryWithNa
*/
public class PluginExtractor {
private static final String NAME_LESSON_I18N_DIRECTORY = "i18n";
private final Path pluginArchive;
private final List<String> classes = Lists.newArrayList();
private final List<Path> files = new ArrayList<>();
private final List<Path> properties = new ArrayList<>();
/**
* <p>Constructor for PluginExtractor.</p>
*
* @param pluginArchive a {@link java.nio.file.Path} object.
*/
public PluginExtractor(Path pluginArchive) {
this.pluginArchive = pluginArchive;
}
public Plugin extractJarFile(final File archive, final File targetDirectory) throws IOException {
ZipFile zipFile = new ZipFile(archive);
Plugin plugin = new Plugin();
try {
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
final ZipEntry zipEntry = entries.nextElement();
if (shouldProcessFile(zipEntry)) {
boolean processed = processClassFile(zipEntry);
/**
* <p>extract.</p>
*
if (!processed) {
* @param target a {@link java.nio.file.Path} object.
*/
public void extract(final Path target) {
try (FileSystem zip = createZipFileSystem()) {
final Path root = zip.getPath("/");
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (file.toString().endsWith(".class")) {
classes.add(file.toString());
processed = processPropertyFile(zipFile, zipEntry, targetDirectory);
}
if (fileEndsWith(file, ".properties") && hasParentDirectoryWithName(file,
NAME_LESSON_I18N_DIRECTORY)) {
properties.add(Files
.copy(file, createDirsIfNotExists(Paths.get(target.toString(), file.toString())),
REPLACE_EXISTING));
if (!processed) {
processFile(plugin, zipFile, zipEntry, targetDirectory);
}
files.add(Files.copy(file, createDirsIfNotExists(Paths.get(target.toString(), file.toString())),
REPLACE_EXISTING));
return FileVisitResult.CONTINUE;
}
});
} catch (Exception e) {
new PluginLoadingFailure(format("Unable to extract: %s", pluginArchive.getFileName()), e);
}
} finally {
plugin.findLesson(this.classes);
if (plugin.getLesson().isPresent()) {
plugin.rewritePaths(targetDirectory.toPath());
}
zipFile.close();
}
return plugin;
}
private void processFile(Plugin plugin, ZipFile zipFile, ZipEntry zipEntry, File targetDirectory)
throws IOException {
final File targetFile = new File(targetDirectory, zipEntry.getName());
copyFile(zipFile, zipEntry, targetFile, false);
plugin.loadFiles(targetFile.toPath(), true);
}
private boolean processPropertyFile(ZipFile zipFile, ZipEntry zipEntry, File targetDirectory)
throws IOException {
if (zipEntry.getName().endsWith(".properties")) {
final File targetFile = new File(targetDirectory, zipEntry.getName());
copyFile(zipFile, zipEntry, targetFile, true);
return true;
}
return false;
}
private boolean processClassFile(ZipEntry zipEntry) {
if (zipEntry.getName().endsWith(".class")) {
classes.add(zipEntry.getName());
return true;
}
return false;
}
private boolean shouldProcessFile(ZipEntry zipEntry) {
return !zipEntry.isDirectory() && !zipEntry.getName().startsWith("META-INF");
}
private File copyFile(ZipFile zipFile, ZipEntry zipEntry, File targetFile, boolean append) throws IOException {
Files.createParentDirs(targetFile);
try (FileOutputStream fos = new FileOutputStream(targetFile, append)) {
Streams.copy(zipFile.getInputStream(zipEntry), fos, true);
}
return targetFile;
}
/**
@ -92,18 +111,4 @@ public class PluginExtractor {
public List<Path> getFiles() {
return this.files;
}
/**
* <p>Getter for the field <code>properties</code>.</p>
*
* @return a {@link java.util.List} object.
*/
public List<Path> getProperties() {
return this.properties;
}
private FileSystem createZipFileSystem() throws Exception {
final URI uri = URI.create("jar:file:" + pluginArchive.toUri().getPath());
return FileSystems.newFileSystem(uri, new HashMap<String, Object>());
}
}

55
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginFileUtils.java
View File

@ -2,17 +2,15 @@ package org.owasp.webgoat.plugins;
import com.google.common.base.Preconditions;
import org.apache.commons.io.IOUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/**
* <p>PluginFileUtils class.</p>
@ -79,24 +77,6 @@ public class PluginFileUtils {
return p;
}
/**
* <p>getFilesInDirectory.</p>
*
* @param directory a {@link java.nio.file.Path} object.
* @return a {@link java.util.List} object.
* @throws java.io.IOException if any.
*/
public static List<Path> getFilesInDirectory(Path directory) throws IOException {
List<Path> files = new ArrayList<>();
DirectoryStream<Path> dirStream;
dirStream = Files.newDirectoryStream(directory);
for (Path entry : dirStream) {
files.add(entry);
}
dirStream.close();
return files;
}
/**
* <p>replaceInFiles.</p>
*
@ -111,7 +91,7 @@ public class PluginFileUtils {
Preconditions.checkNotNull(files);
for (File file : files) {
replaceInFile(replace, with, Paths.get(file.toURI()));
replaceInFile(replace, with, file);
}
}
@ -123,31 +103,16 @@ public class PluginFileUtils {
* @param file a {@link java.nio.file.Path} object.
* @throws java.io.IOException if any.
*/
public static void replaceInFile(String replace, String with, Path file) throws IOException {
public static void replaceInFile(String replace, String with, File file) throws IOException {
Preconditions.checkNotNull(replace);
Preconditions.checkNotNull(with);
Preconditions.checkNotNull(file);
byte[] fileAsBytes = Files.readAllBytes(file);
String fileAsString = new String(fileAsBytes);
fileAsString = fileAsString.replaceAll(replace, with);
Files.write(file, fileAsString.getBytes());
}
/**
* <p>writeFile.</p>
*
* @param targetFile a {@link java.nio.file.Path} object.
* @param bytes an array of byte.
* @param options a {@link java.nio.file.OpenOption} object.
* @throws java.io.IOException if any.
*/
public static void writeFile(Path targetFile, byte[] bytes, OpenOption... options) throws IOException {
createDirsIfNotExists(targetFile.getParent());
if (!Files.exists(targetFile)) {
Files.createFile(targetFile);
String fileAsString = "";
try (FileInputStream fis = new FileInputStream(file);) {
fileAsString = IOUtils.toString(fis, StandardCharsets.UTF_8.name());
fileAsString = fileAsString.replaceAll(replace, with);
}
Files.write(targetFile, bytes, options);
Files.write(file.toPath(), fileAsString.getBytes());
}
}

32
webgoat-container/src/main/java/org/owasp/webgoat/plugins/PluginsLoader.java
View File

@ -6,13 +6,13 @@ import org.owasp.webgoat.classloader.PluginClassLoader;
import org.owasp.webgoat.util.LabelProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.ResourceUtils;
import java.io.IOException;
import java.net.URL;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
@ -88,39 +88,33 @@ public class PluginsLoader implements Runnable {
private List<Plugin> processPlugins(List<URL> jars, boolean reload) throws Exception {
final List<Plugin> plugins = Lists.newArrayList();
final ExecutorService executorService = Executors.newFixedThreadPool(20);
final CompletionService<PluginExtractor> completionService = new ExecutorCompletionService<>(executorService);
final List<Callable<PluginExtractor>> callables = extractJars(jars);
final ExecutorService executorService = Executors.newFixedThreadPool(10);
final CompletionService<Plugin> completionService = new ExecutorCompletionService<>(executorService);
final List<Callable<Plugin>> callables = extractJars(jars);
for (Callable<PluginExtractor> s : callables) {
for (Callable<Plugin> s : callables) {
completionService.submit(s);
}
int n = callables.size();
for (int i = 0; i < n; i++) {
PluginExtractor extractor = completionService.take().get();
Plugin plugin = new Plugin(pluginTarget, extractor.getClasses());
Plugin plugin = completionService.take().get();
if (plugin.getLesson().isPresent()) {
PluginFileUtils.createDirsIfNotExists(pluginTarget);
plugin.loadFiles(extractor.getFiles(), reload);
plugin.loadProperties(extractor.getProperties());
plugin.rewritePaths(pluginTarget);
plugins.add(plugin);
}
}
LabelProvider.refresh();
LabelProvider.updatePluginResources(pluginTarget.resolve("plugin/i18n/WebGoatLabels.properties"));
return plugins;
}
private List<Callable<PluginExtractor>> extractJars(List<URL> jars) {
List<Callable<PluginExtractor>> extractorCallables = Lists.newArrayList();
private List<Callable<Plugin>> extractJars(List<URL> jars) {
List<Callable<Plugin>> extractorCallables = Lists.newArrayList();
for (final URL jar : jars) {
extractorCallables.add(new Callable<PluginExtractor>() {
extractorCallables.add(new Callable<Plugin>() {
@Override
public PluginExtractor call() throws Exception {
PluginExtractor extractor = new PluginExtractor(Paths.get(jar.toURI()));
extractor.extract(pluginTarget);
return extractor;
public Plugin call() throws Exception {
PluginExtractor extractor = new PluginExtractor();
return extractor.extractJarFile(ResourceUtils.getFile(jar), pluginTarget.toFile());
}
});
}

5
webgoat-container/src/main/java/org/owasp/webgoat/util/LabelProvider.java
View File

@ -6,7 +6,6 @@ import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.core.io.UrlResource;
import org.springframework.stereotype.Component;
import org.springframework.util.DefaultPropertiesPersister;
import javax.inject.Singleton;
import java.net.MalformedURLException;
@ -67,9 +66,6 @@ public class LabelProvider {
labels.setFallbackToSystemLocale(false);
labels.setUseCodeAsDefaultMessage(true);
pluginLabels.setParentMessageSource(labels);
pluginLabels.setPropertiesPersister(new DefaultPropertiesPersister() {
});
}
/**
@ -96,7 +92,6 @@ public class LabelProvider {
return Thread.currentThread().getContextClassLoader();
}
});
}
/**
* <p>refresh.</p>

1
webgoat-container/src/test/java/org/owasp/webgoat/plugins/WebGoatIT.java
View File

@ -232,7 +232,6 @@ public class WebGoatIT implements SauceOnDemandSessionIdProvider {
@Test
public void testStartMvc() {
driver.get(baseWebGoatUrl + "/start.mvc");
WebDriverWait wait = new WebDriverWait(driver, 15); // wait for a maximum of 15 seconds

1
webgoat-container/src/test/java/org/owasp/webgoat/util/LabelProviderTest.java
View File

@ -22,7 +22,6 @@ public class LabelProviderTest {
public void loadingPluginLabels() throws IOException {
LabelProvider labelProvider = new LabelProvider();
labelProvider.updatePluginResources(new ClassPathResource("log4j.properties").getFile().toPath());
LabelProvider.refresh();
assertThat(labelProvider.get(Locale.ENGLISH, "LessonCompleted"), CoreMatchers.equalTo(
"Congratulations. You have successfully completed this lesson."));
assertThat(labelProvider.get(Locale.ENGLISH, "log4j.appender.CONSOLE.Target"), CoreMatchers.equalTo(