This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository eugene. See http://git.nuiton.org/eugene.git commit 738ce0998d0f751142911b30c924126c202bdc89 Author: Tony CHEMIT <chemit@codelutin.com> Date: Sun May 24 15:11:45 2015 +0200 introduce FileGrabber API (refs #2937) --- .../java/org/nuiton/eugene/writer/FileGrabber.java | 22 ++ .../eugene/writer/FileGrabberFromClassPath.java | 285 +++++++++++++++++++++ .../eugene/writer/FileGrabberFromDirectory.java | 133 ++++++++++ 3 files changed, 440 insertions(+) diff --git a/eugene/src/main/java/org/nuiton/eugene/writer/FileGrabber.java b/eugene/src/main/java/org/nuiton/eugene/writer/FileGrabber.java new file mode 100644 index 0000000..12f929d --- /dev/null +++ b/eugene/src/main/java/org/nuiton/eugene/writer/FileGrabber.java @@ -0,0 +1,22 @@ +package org.nuiton.eugene.writer; + +import java.io.File; +import java.io.IOException; +import java.util.Set; + +/** + * To grab files to treate. + * + * Created on 5/24/15. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 3.0 + */ +public interface FileGrabber { + + void addFilesToTreate(File extractDirectory, + String inputDirectory, + Set<String> includePatterns, + ChainedFileWriterData result) throws IOException; + +} diff --git a/eugene/src/main/java/org/nuiton/eugene/writer/FileGrabberFromClassPath.java b/eugene/src/main/java/org/nuiton/eugene/writer/FileGrabberFromClassPath.java new file mode 100644 index 0000000..9437d1a --- /dev/null +++ b/eugene/src/main/java/org/nuiton/eugene/writer/FileGrabberFromClassPath.java @@ -0,0 +1,285 @@ +package org.nuiton.eugene.writer; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.util.FileUtil; +import org.nuiton.util.Resource; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Created on 5/24/15. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 3.0 + */ +public class FileGrabberFromClassPath implements FileGrabber { + + /** Logger. */ + private static final Log log = LogFactory.getLog(FileGrabberFromClassPath.class); + + private final ChainedFileWriterConfiguration configuration; + + public FileGrabberFromClassPath(ChainedFileWriterConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public void addFilesToTreate(File extractDirectory, + String inputDirectory, + Set<String> includePatterns, + ChainedFileWriterData result) throws IOException { + + Map<File, List<File>> filesByRoot = result.getFilesByRoot(); + Map<File, List<File>> resourcesByFile = result.getResourcesByFile(); + + // final input directory to use + + File realInputDirectory; + + // use the extracted path as input directory, otherwise there + // will have a problem : the incoming inputPath will not be + // an ancestor of his resources, so prefer to use the extracted + // path which fix this problem. + String inputPath = inputDirectory; + if (inputPath.equals("/")) { + realInputDirectory = extractDirectory; + } else { + realInputDirectory = new File(extractDirectory, inputPath.substring(1)); + } + + List<URL> newUrls = getFiles(inputDirectory, includePatterns); + + List<File> files = filesByRoot.get(realInputDirectory); + if (files == null) { + files = new ArrayList<File>(); + filesByRoot.put(realInputDirectory, files); + } + + for (URL url : newUrls) { + + // get the file + File file = extractFileFromClassPath(extractDirectory, url); + + // add the file in reactor + files.add(file); + + // get resources associated with the file + URL resourceFileUrl = getAssociatedResource(url); + + if (resourceFileUrl == null) { + + // no resource associated with the file + if (log.isDebugEnabled()) { + log.debug("[" + file + "] No resource associated."); + } + + } else { + + // get the resource file + File resourceFile = extractFileFromClassPath(extractDirectory, resourceFileUrl); + if (log.isDebugEnabled()) { + log.debug("[" + file + "] Detected resource " + resourceFile); + } + resourcesByFile.put(file, Collections.singletonList(resourceFile)); + + } + + } + + if (CollectionUtils.isNotEmpty(files)) { + + // check that extracted directory exists, or creates it + boolean b = extractDirectory.exists() || extractDirectory.mkdirs(); + if (!b) { + throw new IOException("Could not create directory " + extractDirectory); + } + + } + + } + + protected List<URL> getFiles(String inputPath, Set<String> includePattern) { + + if (CollectionUtils.isEmpty(includePattern)) { + throw new IllegalArgumentException("Must have at least one include pattern"); + } + + List<URL> result = new ArrayList<URL>(); + + // search in class-path + + ClassLoader loader = configuration.getClassLoader(); + + for (String pattern : includePattern) { + + String path = inputPath; + + //FIXME must change the file.separator to / + if (!path.endsWith("/")) { + path += "/"; + } + path += pattern; + + if (path.startsWith("/")) { + path = path.substring(1); + } + + if (log.isDebugEnabled()) { + log.debug("Try to seek class-path file " + path); + } + + if (pattern.contains("*")) { + + // this is a multi-files to search + List<URL> urlList = Resource.getURLs(path, (URLClassLoader) loader); + if (CollectionUtils.isEmpty(urlList)) { + + log.warn("Could not find in class-path files " + path); + } else { + for (URL url : urlList) { + + if (configuration.isVerbose()) { + log.info("Detected class-path file " + url); + } + result.add(url); + } + } + } else { + + // this is a simple unique search, improve performance + // by searching directly in classloader the resource + URL url = loader.getResource(path); + if (url == null) { + + log.warn("Could not find in class-path the file " + path); + } else { + + if (configuration.isVerbose()) { + log.info("Detected class-path file " + url); + } + result.add(url); + } + } + } + return result; + } + + protected URL getAssociatedResource(URL file) throws IOException { + + // obtain the properties files associated with the file + String path = file.toString(); + + String extension = "." + FileUtil.extension(path); + + String filename = FileUtil.basename(path, extension).concat(".properties"); + + if (log.isDebugEnabled()) { + log.info("path of file : " + path); + log.info("path of resource : " + filename); + } + + URL result; + + URL propertiesFile = URI.create(filename).toURL(); + + if (path.startsWith("file:")) { + + //FIXME-tchemit-2015-05-24 Does this case happen here ? + // local file (not from class-path) + // can test directly on resource if it exists + File file1 = new File(propertiesFile.getFile()); + if (file1.exists()) { + + // resource exist, keep it + result = propertiesFile; + + } else { + + result = null; + + } + + } else { + + InputStream in = null; + try { + in = propertiesFile.openStream(); + + // resource exist, keep it + result = propertiesFile; + + } catch (IOException eee) { + + // resource does not exists + log.warn("Could not find resource " + propertiesFile); + result = null; + + } finally { + + if (in != null) { + in.close(); + } + + } + + } + + return result; + + } + + protected File extractFileFromClassPath(File extractDirectory, URL url) throws IOException { + + String path = url.getPath(); + + // case where file is extracted from jar, "!" found into url, ex: + // url: /home/.../agrosyst-api/target/agrosyst-api-1.0.1-SNAPSHOT.jar!/agrosyst.objectmodel + // path: /home/.../agrosyst-services/target/extracted-sources/model/agrosyst.objectmodel + + // case where file is no extracted from jar, "!" not found into url, ex: + // url: /home/.../agrosyst-api/target/classes/agrosyst.objectmodel + // path: /home/.../agrosyst-services/target/extracted-sources/model/agrosyst.objectmodel + + int index = path.indexOf("!"); + if (index == -1) { + // case where file is no extracted from jar: + index = path.lastIndexOf("/") - 1; // -1: because we need to keep the last "/" from path + } + String relativePath = path.substring(index + 1); + + File f = new File(extractDirectory, relativePath); + if (log.isDebugEnabled()) { + log.debug("extract " + url + " to " + f); + } + File parentFile = f.getParentFile(); + + boolean b = parentFile.exists() || parentFile.mkdirs(); + if (!b) { + throw new IOException("Could not create directory " + f); + } + + FileOutputStream out = new FileOutputStream(f); + try { + IOUtils.copy(url.openStream(), out); + } finally { + out.close(); + } + return f; + + } + +} diff --git a/eugene/src/main/java/org/nuiton/eugene/writer/FileGrabberFromDirectory.java b/eugene/src/main/java/org/nuiton/eugene/writer/FileGrabberFromDirectory.java new file mode 100644 index 0000000..3981dfc --- /dev/null +++ b/eugene/src/main/java/org/nuiton/eugene/writer/FileGrabberFromDirectory.java @@ -0,0 +1,133 @@ +package org.nuiton.eugene.writer; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.codehaus.plexus.util.DirectoryScanner; +import org.nuiton.util.FileUtil; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Created on 5/24/15. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 3.0 + */ +public class FileGrabberFromDirectory implements FileGrabber { + + /** Logger. */ + private static final Log log = LogFactory.getLog(FileGrabberFromDirectory.class); + + private final ChainedFileWriterConfiguration configuration; + + public FileGrabberFromDirectory(ChainedFileWriterConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public void addFilesToTreate(File extractDirectory, + String inputDirectory, + Set<String> includePatterns, + ChainedFileWriterData result) throws IOException { + + Map<File, List<File>> filesByRoot = result.getFilesByRoot(); + Map<File, List<File>> resourcesByFile = result.getResourcesByFile(); + + // final input directory to use + + File realInputDirectory = new File(inputDirectory); + + List<File> files = filesByRoot.get(realInputDirectory); + if (files == null) { + files = new ArrayList<File>(); + filesByRoot.put(realInputDirectory, files); + } + + List<File> newFiles = getFiles(inputDirectory, includePatterns); + + for (File file : newFiles) { + + // add the file in reactor + files.add(file); + + // get resources associated with the file + File resourceFile = getAssociatedResource(file); + + if (resourceFile == null) { + + // no resource associated with the file + if (log.isDebugEnabled()) { + log.debug("[" + file + "] No resource associated."); + } + + } else { + + if (configuration.isVerbose() && log.isDebugEnabled()) { + log.debug("[" + file + "] Detected resource " + resourceFile); + } + List<File> resources = new ArrayList<File>(1); + resources.add(resourceFile); + + resourcesByFile.put(file, resources); + + } + + } + + } + + protected List<File> getFiles(String inputPath, Set<String> includePattern) { + + if (CollectionUtils.isEmpty(includePattern)) { + throw new IllegalArgumentException("Must have at least one include pattern"); + } + + List<File> result = new ArrayList<File>(); + + DirectoryScanner ds = new DirectoryScanner(); + File inputDirectory = new File(inputPath); + ds.setBasedir(inputDirectory); + ds.setIncludes(includePattern.toArray(new String[includePattern.size()])); + ds.setExcludes(null); + ds.addDefaultExcludes(); + ds.scan(); + + for (String file : ds.getIncludedFiles()) { + File in = new File(inputDirectory, file); + result.add(in); + } + + return result; + + } + + protected File getAssociatedResource(File file) throws IOException { + + String extension = "." + FileUtil.extension(file.getName()); + + String path = file.getAbsolutePath(); + + String filename = FileUtil.basename(path, extension).concat(".properties"); + + if (log.isDebugEnabled()) { + log.info("path of file : " + path); + log.info("path of resource : " + filename); + } + + File propertiesFile = new File(filename); + + if (!propertiesFile.exists()) { + propertiesFile = null; + } + + return propertiesFile; + + } + +} -- To stop receiving notification emails like this one, please contact nuiton.org SCM administrator <admin+scm@nuiton.org>.