Author: bpoussin Date: 2010-04-28 09:46:33 +0200 (Wed, 28 Apr 2010) New Revision: 1756 Log: simplify multi-threading (again :)), use ParserExecutor directly, remove ParserThread run-its test work, but more test must be done Added: trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/ParserExecutor.java Removed: trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/ParserThread.java Modified: trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/AbstractI18nParserMojo.java Modified: trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/AbstractI18nParserMojo.java =================================================================== --- trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/AbstractI18nParserMojo.java 2010-04-27 16:54:55 UTC (rev 1755) +++ trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/AbstractI18nParserMojo.java 2010-04-28 07:46:33 UTC (rev 1756) @@ -70,7 +70,7 @@ protected abstract File getDefaultBasedir(); /** - * @return a new file parser to be used in the parser consumer thread + * @return a new file parser to be used in the parser consumer parserExecutor * @since 1.2 */ public abstract FileParser newFileParser(); @@ -157,7 +157,7 @@ protected long t0; - ParserThread thread; + ParserExecutor parserExecutor; @Override public boolean isStrictMode() { @@ -201,8 +201,7 @@ } } - // thread is started when we add the first file - thread = new ParserThread(this); + parserExecutor = new ParserExecutor(this); } @Override @@ -272,20 +271,20 @@ String fileName = entry.getBasedir().getAbsolutePath() + File.separator + file1; File file = new File(fileName); - thread.addFile(newFileParser(), file); + parserExecutor.addFile(newFileParser(), file); } } if (getLog().isDebugEnabled()) { - getLog().debug("ask to terminate " + thread); + getLog().debug("ask to terminate " + parserExecutor); } - // all files are send to thread, we ask termination of thread. + // all files are send to parserExecutor, we ask termination of parserExecutor. // this termination treat all sending file before really stop - thread.terminatesAndWaits(); + parserExecutor.terminatesAndWaits(); - List<File> treadedFiles = thread.getTreatedFiles(); - List<File> touchedFiles = thread.getTouchedFiles(); + List<File> treadedFiles = parserExecutor.getTreatedFiles(); + List<File> touchedFiles = parserExecutor.getTouchedFiles(); // Suppression du fichier sauvegarder if (!backupGetter) { @@ -315,7 +314,7 @@ addGetter(); } - thread.clear(); + parserExecutor.clear(); } @Override Added: trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/ParserExecutor.java =================================================================== --- trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/ParserExecutor.java (rev 0) +++ trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/ParserExecutor.java 2010-04-28 07:46:33 UTC (rev 1756) @@ -0,0 +1,303 @@ +/* *##% + * Copyright (c) 2010 poussin. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + *##%*/ + +package org.nuiton.i18n.plugin.parser; + + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.maven.plugin.logging.Log; +import org.nuiton.io.SortedProperties; +import org.nuiton.plugin.PluginHelper; + +/** + * + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class ParserExecutor extends ThreadPoolExecutor + implements I18nParserConfiguration { + + /** + * the incoming configuration (from mojo which contains shared result and + * logger) + */ + protected final I18nParserConfiguration configuration; + + /** list of files consumed */ + protected final List<File> treatedFiles; + + /** list of files touched (says having at least one i18n key) */ + protected final List<File> touchedFiles; + + /** number of files registred to consume */ + protected int nbFiles; + + /** permit to stop thread and executor */ + + public ParserExecutor(I18nParserConfiguration configuration) { + super(8, 10, 1L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); + this.configuration = configuration; + touchedFiles = new ArrayList<File>(); + treatedFiles = new ArrayList<File>(); + } + + @Override + protected void afterExecute(Runnable r, Throwable t) { + super.afterExecute(r, t); + ParserTask i18n = (ParserTask) r; + i18n.registerResult(treatedFiles, touchedFiles, getResult()); + } + + @Override + public boolean isVerbose() { + return getConfiguration().isVerbose(); + } + + @Override + public boolean isSilent() { + return getConfiguration().isSilent(); + } + + @Override + public boolean isShowTouchedFiles() { + return getConfiguration().isShowTouchedFiles(); + } + + @Override + public Log getLog() { + return getConfiguration().getLog(); + } + + @Override + public SortedProperties getResult() { + return getConfiguration().getResult(); + } + + public List<File> getTreatedFiles() { + return treatedFiles; + } + + public List<File> getTouchedFiles() { + return touchedFiles; + } + + + + /** clean internal state after usage of the thread. */ + public void clear() { + treatedFiles.clear(); + touchedFiles.clear(); + nbFiles = 0; + } + + /** + * Add a file to be consumed. + * + * @param parser the parser of the file + * @param files files to parse + */ + public void addFile(FileParser parser, File... files) { + + for (File f : files) { + + nbFiles++; + if (isVerbose()) { + getLog().info("[" + nbFiles + "] " + f); + } + execute(new ParserTask(parser, f)); + } + } + + /** + * Ask the thread to stop. + * <p/> + * It will finish all incoming files (but will not accept more files to + * parse) + * <p/> + * <b>Note:</b> The method does not return until all files are not + * consumed. + */ + public void terminatesAndWaits() throws InterruptedException { + // ask executor to terminate + shutdown(); + + if (isVerbose()) { + if (nbFiles == 0) { + // no file consumed + getLog().info("No file consumed."); + } else { + getLog().info("Will waits until all files (" + nbFiles + + ") are consumed (still " + getNbFilesToTreate() + + " file(s) to consume)"); + } + } + + + try { + // wait until all submited jobs are terminated + // i don't want timeout, i think 2 days is good :) + awaitTermination(2, TimeUnit.DAYS); + } catch (InterruptedException e) { + getLog().error(e); + } + + if (getLog().isDebugEnabled()) { + getLog().debug("A task was consumed, still " + getNbFilesToTreate() + + " file(s) to treate."); + } + + if (isVerbose()) { + getLog().info("Executor is terminated."); + } + } + + protected I18nParserConfiguration getConfiguration() { + return configuration; + } + + protected int getNbFilesToTreate() { + return nbFiles - treatedFiles.size(); + } + + /** + * This is a task to parse a {@link #file}. + * <p/> + * The task will be executed in the executor service created in the thread. + */ + class ParserTask implements Runnable { + + /** the file parser */ + protected final FileParser parser; + + /** the file to parse */ + protected final File file; + + /** starting time */ + protected long startingTime; + + /** ending time */ + protected long endingTime; + + ParserTask(FileParser parser, File file) { + this.parser = parser; + this.file = file; + } + + @Override + public void run() { + startingTime = System.nanoTime(); + if (getLog().isDebugEnabled()) { + getLog().debug("starting action for " + file); + } + try { + + parser.parseFile(file); + + } catch (IOException e) { + if (getLog().isErrorEnabled()) { + getLog().error("could not parse file " + file, e); + } + } finally { + + if (getLog().isDebugEnabled()) { + getLog().debug("ending action for " + file); + } + endingTime = System.nanoTime(); + } + } + + @Override + public String toString() { + return super.toString() + " - " + file; + } + + protected File getFile() { + return file; + } + + protected long getDelay() { + return endingTime - startingTime; + } + + protected void destroy() { + parser.destroy(); + } + + @Override + protected void finalize() throws Throwable { + super.finalize(); + destroy(); + } + + /** + * Register the result of the parsing of the {@link #file} after {@link + * #run()} method was invoked. + * <p/> + * This method should be invoked by the executor as an ending hook. + * + * @param treatedFiles list of files already treated + * @param touchedFiles list of files already touched + * @param result shared result. + */ + protected synchronized void registerResult(List<File> treatedFiles, + List<File> touchedFiles, + SortedProperties result) { + + try { + treatedFiles.add(file); + if (getLog().isDebugEnabled()) { + String delay = PluginHelper.convertTime(getDelay()); + getLog().debug("[" + treatedFiles.size() + "] " + file + + " in " + delay); + } + + if (parser.isTouched()) { + + // mark file as touched + touchedFiles.add(file); + + if (isShowTouchedFiles()) { + getLog().info("touch " + file); + } + if (isVerbose()) { + String delay = PluginHelper.convertTime(getDelay()); + getLog().info("[" + treatedFiles.size() + "] touchs " + + file + " in " + delay); + } + + // merge file result with + // merge result + result.putAll(parser.getResult()); + } + } finally { + + // destroy runner + destroy(); + } + } + } + +} Deleted: trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/ParserThread.java =================================================================== --- trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/ParserThread.java 2010-04-27 16:54:55 UTC (rev 1755) +++ trunk/maven-i18n-plugin/src/main/java/org/nuiton/i18n/plugin/parser/ParserThread.java 2010-04-28 07:46:33 UTC (rev 1756) @@ -1,393 +0,0 @@ -/* - * #%L - * I18n :: Maven Plugin - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2007 - 2010 CodeLutin - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Lesser Public License for more details. - * - * You should have received a copy of the GNU General Lesser Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/lgpl-3.0.html>. - * #L% - */ - -package org.nuiton.i18n.plugin.parser; - -import org.apache.maven.plugin.logging.Log; -import org.nuiton.io.SortedProperties; -import org.nuiton.plugin.PluginHelper; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - * A thread to launch parser on files. - * - * @author tchemit <chemit@codelutin.com> - * @since 1.2 - */ -public class ParserThread extends Thread implements I18nParserConfiguration { - - /** - * the incoming configuration (from mojo which contains shared result and - * logger) - */ - protected final I18nParserConfiguration configuration; - - /** queue of tasks to consume */ - protected final BlockingQueue<ParserTask> tasks; - - /** list of files consumed */ - protected final List<File> treatedFiles; - - /** list of files touched (says having at least one i18n key) */ - protected final List<File> touchedFiles; - - /** number of files registred to consume */ - protected int nbFiles; - - /** permit to stop thread and executor */ - boolean quitAsked = false; - - public ParserThread(I18nParserConfiguration configuration) { - super(ParserThread.class.getSimpleName()); - this.configuration = configuration; - tasks = new LinkedBlockingQueue<ParserTask>(); - touchedFiles = new ArrayList<File>(); - treatedFiles = new ArrayList<File>(); - } - - @Override - public boolean isVerbose() { - return getConfiguration().isVerbose(); - } - - @Override - public boolean isSilent() { - return getConfiguration().isSilent(); - } - - @Override - public boolean isShowTouchedFiles() { - return getConfiguration().isShowTouchedFiles(); - } - - @Override - public Log getLog() { - return getConfiguration().getLog(); - } - - @Override - public SortedProperties getResult() { - return getConfiguration().getResult(); - } - - public List<File> getTreatedFiles() { - return treatedFiles; - } - - public List<File> getTouchedFiles() { - return touchedFiles; - } - - /** - * Add a file to be consumed. - * - * @param parser the parser of the file - * @param files files to parse - */ - public void addFile(FileParser parser, File... files) { - - for (File f : files) { - - nbFiles++; - if (isVerbose()) { - getLog().info("[" + nbFiles + "] " + f); - } - tasks.offer(new ParserTask(parser, f)); - if (!isAlive()) { - // we are sure that we have 1 file to treat, we start the thread - start(); - } - } - } - - /** - * Ask the thread to stop. - * <p/> - * It will finish all incoming files (but will not accept more files to - * parse) - * <p/> - * <b>Note:</b> The method does not return until all files are not - * consumed. - */ - public void terminatesAndWaits() throws InterruptedException { - quitAsked = true; - // wait thread termination - this.join(); - } - - @Override - public void run() { - - // create pool executor - ExecutorService executor = createExecutor(); - - // only quit loop if no more task in queue and quitAsked is true - while (!(quitAsked && tasks.size() == 0)) { - - // waiting for a new file to treate - ParserTask runner; - try { - // ask task on queue, wait 500ms until there is 1 file - // don't use take(), because we must check when user switch - // quitAsked flag with terminatesAndWaits() - runner = tasks.poll(500, TimeUnit.MILLISECONDS); - if (runner == null) { - // no file in queue, go to check quitAsked flag - continue; - } - } catch (InterruptedException e) { - if (getLog().isErrorEnabled()) { - getLog().error(e); - } - break; - } - - if (getLog().isDebugEnabled()) { - getLog().debug("consume " + runner); - } - executor.execute(runner); - - if (getLog().isDebugEnabled()) { - getLog().debug("after consume " + runner); - } - } - - // something append to ask the termination of thread, we stop executor - // ask executor to terminate - executor.shutdown(); - - if (isVerbose()) { - if (nbFiles == 0) { - // no file consumed - getLog().info("No file consumed."); - } else { - getLog().info("Will waits until all files (" + nbFiles + - ") are consumed (still " + getNbFilesToTreate() + - " file(s) to consume)"); - } - } - - - try { - // wait until all submited jobs are terminated - // i don't want timeout, i think 2 days is good :) - executor.awaitTermination(2, TimeUnit.DAYS); - } catch (InterruptedException e) { - getLog().error(e); - } - - if (getLog().isDebugEnabled()) { - getLog().debug("A task was consumed, still " + getNbFilesToTreate() + - " file(s) to treate."); - } - - if (isVerbose()) { - getLog().info("Executor is terminated, will release thread lock."); - } - } - - /** clean internal state after usage of the thread. */ - public void clear() { - treatedFiles.clear(); - touchedFiles.clear(); - nbFiles = 0; - } - - /** - * Creates the {@link ExecutorService} which will consumed i8n tasks. - * - * @return the instanciated executor - */ - protected ExecutorService createExecutor() { - - // creates thread factory - ThreadFactory factory = - new ThreadFactory() { - final ThreadFactory defaultFactory = - Executors.defaultThreadFactory(); - - public Thread newThread(Runnable r) { - Thread thread = - defaultFactory.newThread(r); - thread.setName(ParserThread.this + "-" + - thread.getName()); - return thread; - } - }; - - // create pool executor - ExecutorService executor = new ThreadPoolExecutor( - 8, 10, - 1L, TimeUnit.SECONDS, - new LinkedBlockingQueue<Runnable>(), - factory) { - - @Override - protected void afterExecute(Runnable r, Throwable t) { - super.afterExecute(r, t); - ParserTask i18n = (ParserTask) r; - i18n.registerResult(treatedFiles, touchedFiles, getResult()); - } - }; - return executor; - } - - protected I18nParserConfiguration getConfiguration() { - return configuration; - } - - protected int getNbFilesToTreate() { - return nbFiles - treatedFiles.size(); - } - - /** - * This is a task to parse a {@link #file}. - * <p/> - * The task will be executed in the executor service created in the thread. - */ - class ParserTask implements Runnable { - - /** the file parser */ - protected final FileParser parser; - - /** the file to parse */ - protected final File file; - - /** starting time */ - protected long startingTime; - - /** ending time */ - protected long endingTime; - - ParserTask(FileParser parser, File file) { - this.parser = parser; - this.file = file; - } - - @Override - public void run() { - startingTime = System.nanoTime(); - if (getLog().isDebugEnabled()) { - getLog().debug("starting action for " + file); - } - try { - - parser.parseFile(file); - - } catch (IOException e) { - if (getLog().isErrorEnabled()) { - getLog().error("could not parse file " + file, e); - } - } finally { - - if (getLog().isDebugEnabled()) { - getLog().debug("ending action for " + file); - } - endingTime = System.nanoTime(); - } - } - - @Override - public String toString() { - return super.toString() + " - " + file; - } - - protected File getFile() { - return file; - } - - protected long getDelay() { - return endingTime - startingTime; - } - - protected void destroy() { - parser.destroy(); - } - - @Override - protected void finalize() throws Throwable { - super.finalize(); - destroy(); - } - - /** - * Register the result of the parsing of the {@link #file} after {@link - * #run()} method was invoked. - * <p/> - * This method should be invoked by the executor as an ending hook. - * - * @param treatedFiles list of files already treated - * @param touchedFiles list of files already touched - * @param result shared result. - */ - protected synchronized void registerResult(List<File> treatedFiles, - List<File> touchedFiles, - SortedProperties result) { - - try { - treatedFiles.add(file); - if (getLog().isDebugEnabled()) { - String delay = PluginHelper.convertTime(getDelay()); - getLog().debug("[" + treatedFiles.size() + "] " + file + - " in " + delay); - } - - if (parser.isTouched()) { - - // mark file as touched - touchedFiles.add(file); - - if (isShowTouchedFiles()) { - getLog().info("touch " + file); - } - if (isVerbose()) { - String delay = PluginHelper.convertTime(getDelay()); - getLog().info("[" + treatedFiles.size() + "] touchs " + - file + " in " + delay); - } - - // merge file result with - // merge result - result.putAll(parser.getResult()); - } - } finally { - - // destroy runner - destroy(); - } - } - } -}