Author: tchemit Date: 2008-02-02 15:47:56 +0000 (Sat, 02 Feb 2008) New Revision: 580 Added: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseLoadTestCase.java trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseTestCase.java trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThread.java trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsTestCase.java trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneSimpleTest.java Removed: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java Log: deplacement des tests unitaires dans le m?\195?\170me paquetage que les classes testees Added: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseLoadTestCase.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseLoadTestCase.java (rev 0) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseLoadTestCase.java 2008-02-02 15:47:56 UTC (rev 580) @@ -0,0 +1,183 @@ +/* +* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* ##% */ +package fr.cemagref.simexplorer.is.storage.database.lucene; + +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import junit.framework.TestCase; +import fr.cemagref.simexplorer.is.entities.metadata.MetaData; +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; +import fr.cemagref.simexplorer.is.storage.MetaDataGenerator; + +public class LuceneDatabaseLoadTestCase extends TestCase { + + private Database database; + private Random r = new Random(); + private MetaDataGenerator mdg = new MetaDataGenerator(); + + private static boolean stringsInit = false; + private static int cs; + private static String[] randomstrings; + + private static String validChars = "abcdefghijklmnopqrstuvwxyz"; + + private String generateString(int length) { + StringBuffer result = new StringBuffer(); + + for (int i = 0; i < length; i++) { + char c = validChars.charAt(r.nextInt(validChars.length())); + result.append(c); + } + + return result.toString(); + } + + private void initializeStrings() { + + if (!stringsInit) { + cs = 150000; + randomstrings = new String[cs]; + for (int i = 0; i < randomstrings.length; i++) { + randomstrings[i] = generateString(2 + r.nextInt(6)); + } + stringsInit = true; + } + + } + + public LuceneDatabaseLoadTestCase() { + super(); + initializeStrings(); + } + + public LuceneDatabaseLoadTestCase(String name) { + super(name); + initializeStrings(); + } + + @Override + protected void setUp() throws Exception { + super.setUp(); + + database = new LuceneDatabase(); + // database.open(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + database.close(); + } + + public void testMassInsert() throws Exception { + database.open(true); + + long contentSize = 0; + + int c = 25000; + Date begin = new Date(); + + StringReader reader = null; + List<Reader> readers = new ArrayList<Reader>(); + + for (int i = 0; i < c; i++) { + MetaData[] mes = mdg.generateVersionnedMetaDataEntity(); + for (int j = 0; j < mes.length; j++) { + + int wordcount = 10000 + r.nextInt(20000); + + StringBuffer sb = new StringBuffer(); + for (int k = 0; k < wordcount; k++) { + sb.append(randomstrings[r.nextInt(cs)]).append(" "); + } + + String buf = sb.toString(); + contentSize += buf.length(); + reader = new StringReader(buf); + readers.clear(); + readers.add(reader); + + database.insertElement(mes[j], readers); + } + if (i > 0 && i % 100 == 0) { + + database.commit(); + + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Insert " + i + " : Time taken : " + time + + "ms"); + double timePerElement = time / (4 * i); + System.out.println("Insert " + i + + " : Time taken per element : " + timePerElement + + "ms"); + + System.out.println("Content size : " + contentSize); + } + + } + database.commit(); + + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Insert : Time taken : " + time + "ms"); + double timePerElement = time / (4 * c); + System.out.println("Insert : Time taken per element : " + + timePerElement + "ms"); + System.out.println("Content size : " + contentSize); + } + + public void testMassSearchPaginated() throws Exception { + database.open(); + + Date begin = new Date(); + + int csearch = 100; + + for (int i = 0; i < csearch; i++) { + String key = randomstrings[r.nextInt(randomstrings.length)] + "*"; + + int count = database.findElementsByContentSearchCount(key, false); + System.out.println(key + " : " + count); + + int start = count / 2; + int length = Math.min(200, Math.min(count, Math.max(1, count / 4))); + + Set<MetaData> metaDatas = database + .findElementsByContentSearch(key, false, start, length, 1); + + System.out.println(key + " (" + start + " " + length + ") : " + + metaDatas.size()); + + } + Date end = new Date(); + long time = end.getTime() - begin.getTime(); + System.out.println("Search : Time taken : " + time + "ms"); + double timePerElement = time / csearch; + System.out.println("Search : Time taken per element : " + + timePerElement + "ms"); + + } +} Added: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseTestCase.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseTestCase.java (rev 0) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseTestCase.java 2008-02-02 15:47:56 UTC (rev 580) @@ -0,0 +1,139 @@ +/* +* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* ##% */ +package fr.cemagref.simexplorer.is.storage.database.lucene; + +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import junit.framework.TestCase; +import fr.cemagref.simexplorer.is.entities.metadata.MetaData; +import fr.cemagref.simexplorer.is.entities.metadata.Version; +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; +import fr.cemagref.simexplorer.is.storage.MetaDataGenerator; + +public class LuceneDatabaseTestCase extends TestCase { + + private Database database; + // private ElementGenerator elementGenerator; + private MetaDataGenerator mdg = new MetaDataGenerator(); + private Random r = new Random(); + + @Override + protected void setUp() throws Exception { + super.setUp(); + + database = new LuceneDatabase(); + database.open(true); + // elementGenerator = new ElementGenerator(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + database.close(); + } + + public void testInsertElementGetElement() throws Exception { + MetaData me = mdg.generateMetaDataEntity(); + database.insertElement(me, new ArrayList<Reader>()); + database.commit(); + MetaData me2 = database.getElement(me.getUuid(), me.getVersion()); + assertEquals(me.getDescription(), me2.getDescription()); + database.insertElement(me, new ArrayList<Reader>()); + database.commit(); + } + + public void testIndexAddFindElementsByContentSearch() throws Exception { + MetaData me = mdg.generateMetaDataEntity(); + + String keyphrase = Long.toHexString(r.nextLong()); + + StringReader reader = new StringReader("ogijeriog ergijergioj " + + keyphrase + " egiojergij ergijerogij"); + + List<Reader> readers = new ArrayList<Reader>(); + readers.add(reader); + + database.insertElement(me, readers); + database.commit(); + + int mesCount = database.findElementsByContentSearchCount(keyphrase, + true); + + assertEquals(1, mesCount); + } + + public void testGetVersions() throws Exception { + MetaData[] mes = mdg.generateVersionnedMetaDataEntity(); + for (int i = 0; i < mes.length; i++) { + database.insertElement(mes[i], new ArrayList<Reader>()); + } + database.commit(); + + List<Version> versions = database.getVersions(mes[0].getUuid()); + assertEquals(mes.length, versions.size()); + Collections.sort(versions); + assertEquals("1.0", versions.get(0).toString()); + assertEquals("1.1", versions.get(1).toString()); + assertEquals("1.2", versions.get(2).toString()); + assertEquals("2.0", versions.get(3).toString()); + } + + public void testFindElementsById() throws Exception { + MetaData[] mes = mdg.generateVersionnedMetaDataEntity(); + for (int i = 0; i < mes.length; i++) { + database.insertElement(mes[i], new ArrayList<Reader>()); + } + database.commit(); + + Set<MetaData> elements = database.findElementsById(mes[0] + .getUuid(), false); + assertEquals(elements.size(), mes.length); + } + + public void testFindElementsByProperties() throws Exception { + MetaData me = mdg.generateMetaDataEntity(); + + database.insertElement(me, new ArrayList<Reader>()); + database.commit(); + + } + + public void testDeleteElements() { + fail("Not yet implemented"); + } + + public void testDeleteElementStringVersion() { + fail("Not yet implemented"); + } + + public void testGetElementLatestVersion() { + fail("Not yet implemented"); + } + + public void testDeleteElementMetaDataEntity() { + fail("Not yet implemented"); + } + +} Added: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThread.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThread.java (rev 0) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThread.java 2008-02-02 15:47:56 UTC (rev 580) @@ -0,0 +1,114 @@ +/* +* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* ##% */ +package fr.cemagref.simexplorer.is.storage.database.lucene; + +import java.io.Reader; +import java.util.ArrayList; +import java.util.Random; +import java.util.Set; + +import fr.cemagref.simexplorer.is.entities.metadata.MetaData; +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.MetaDataGenerator; + +public class LuceneDatabaseThread extends Thread { + + private Database database; + private int idThreadLucene; + + private MetaDataGenerator mdg = new MetaDataGenerator(); + + /** + * @return the database + */ + public Database getDatabase() { + return database; + } + + /** + * @param database + * the database to set + */ + public void setDatabase(Database database) { + this.database = database; + } + + @Override + public void run() { + int npass = 10; + String[] ids = new String[npass]; + for (int i = 0; i < npass; i++) { + try { + ids[i] = insertElements(i); + } catch (Exception e) { + System.err.println(idThreadLucene + " - insert : " + e); + } + } + + for (int i = 0; i < npass; i++) { + try { + searchElements(ids[i]); + } catch (Exception e) { + System.err.println(idThreadLucene + " - search : " + e); + } + } + + } + + private void searchElements(String id) throws Exception { + Set<MetaData> mde = database.findElementsById(id, true); + if (mde.isEmpty()) { + System.err.println(idThreadLucene + " - " + id); + } + } + + private String insertElements(int ipass) throws Exception { + + String id = null; + + int ncycles = 5 + (new Random()).nextInt(20); + + for (int j = 0; j < ncycles; j++) { + MetaData[] mes = mdg.generateVersionnedMetaDataEntity(); + for (MetaData me : mes) { + me.setParentDataUuid(Integer.toString(idThreadLucene)); + database.insertElement(me, new ArrayList<Reader>()); + id = me.getUuid(); + } + } + System.out.print(idThreadLucene + " "); + database.commit(); + return id; + } + + /** + * @return the idThreadLucene + */ + public int getIdThreadLucene() { + return idThreadLucene; + } + + /** + * @param idThreadLucene + * the idThreadLucene to set + */ + public void setIdThreadLucene(int idThreadLucene) { + this.idThreadLucene = idThreadLucene; + } + +} Added: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsTestCase.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsTestCase.java (rev 0) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneDatabaseThreadsTestCase.java 2008-02-02 15:47:56 UTC (rev 580) @@ -0,0 +1,59 @@ +/* +* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* ##% */ +package fr.cemagref.simexplorer.is.storage.database.lucene; + +import junit.framework.TestCase; +import fr.cemagref.simexplorer.is.storage.database.Database; +import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; + +public class LuceneDatabaseThreadsTestCase extends TestCase { + + private Database database; + + @Override + protected void setUp() throws Exception { + super.setUp(); + database = new LuceneDatabase(); + database.open(true); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + database.close(); + } + + public void testLuceneDatabaseThreads() throws Exception { + + int nthreads = 10; + + LuceneDatabaseThread[] threads = new LuceneDatabaseThread[nthreads]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new LuceneDatabaseThread(); + threads[i].setDatabase(database); + threads[i].setIdThreadLucene(i); + threads[i].start(); + } + + for (int i = 0; i < threads.length; i++) { + threads[i].join(); + } + + } + +} Added: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneSimpleTest.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneSimpleTest.java (rev 0) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/database/lucene/LuceneSimpleTest.java 2008-02-02 15:47:56 UTC (rev 580) @@ -0,0 +1,89 @@ +/* +* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +* ##% */ +package fr.cemagref.simexplorer.is.storage.database.lucene; + +import junit.framework.TestCase; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; + +public class LuceneSimpleTest extends TestCase { + + /* + * public void testSearch() throws Exception { Searcher searcher; searcher = + * new IndexSearcher("/home/landais/lucene/index.big"); Query query = new + * TermQuery(new Term("simexplorer.type", "LuceneDatabaseTestCase")); + * + * Sort sort = new Sort("simexplorer.creationdate", true); Hits hits = + * searcher.search(query, sort); for (int i = 0; i < 50; i++) { Document doc = + * hits.doc(i); System.out.println(doc.get("simexplorer.creationdate")); } + * System.out.println(hits.length()); } + */ + + public void testLuceneSimple() throws Exception { + + IndexWriter writer; + Searcher searcher; + Analyzer analyser; + Query query; + + /* + * String indexDir = "./index/"; searcher = new IndexSearcher(indexDir); + * query = new TermQuery(new Term("simexplorer.type", + * "LuceneDatabaseTestCase")); Filter filter = new QueryFilter(query); + * System.out.println(searcher.search(query, null, 10).totalHits); + */ + + String indexDir = "./indextest/"; + analyser = new SimpleAnalyzer(); + Directory dir = FSDirectory.getDirectory(indexDir); + writer = new IndexWriter(dir, true, analyser, true); + + searcher = new IndexSearcher(indexDir); + + Document doc = new Document(); + + doc.add(new Field("field1", "abcd", Field.Store.YES, + Field.Index.UN_TOKENIZED)); + doc.add(new Field("field2", "dcba", Field.Store.YES, + Field.Index.UN_TOKENIZED)); + + writer.addDocument(doc); + + query = new TermQuery(new Term("field1", "abcd")); + + System.out.println(searcher.search(query).length()); + writer.flush(); + System.out.println(searcher.search(query).length()); + writer.close(); + System.out.println(searcher.search(query).length()); + searcher = new IndexSearcher(indexDir); + System.out.println(searcher.search(query).length()); + } + +} Deleted: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java 2008-02-02 15:04:22 UTC (rev 579) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseLoadTestCase.java 2008-02-02 15:47:56 UTC (rev 580) @@ -1,183 +0,0 @@ -/* -* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -* ##% */ -package fr.cemagref.simexplorer.is.storage.test; - -import java.io.Reader; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Random; -import java.util.Set; - -import junit.framework.TestCase; -import fr.cemagref.simexplorer.is.entities.metadata.MetaData; -import fr.cemagref.simexplorer.is.storage.database.Database; -import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; -import fr.cemagref.simexplorer.is.storage.MetaDataGenerator; - -public class LuceneDatabaseLoadTestCase extends TestCase { - - private Database database; - private Random r = new Random(); - private MetaDataGenerator mdg = new MetaDataGenerator(); - - private static boolean stringsInit = false; - private static int cs; - private static String[] randomstrings; - - private static String validChars = "abcdefghijklmnopqrstuvwxyz"; - - private String generateString(int length) { - StringBuffer result = new StringBuffer(); - - for (int i = 0; i < length; i++) { - char c = validChars.charAt(r.nextInt(validChars.length())); - result.append(c); - } - - return result.toString(); - } - - private void initializeStrings() { - - if (!stringsInit) { - cs = 150000; - randomstrings = new String[cs]; - for (int i = 0; i < randomstrings.length; i++) { - randomstrings[i] = generateString(2 + r.nextInt(6)); - } - stringsInit = true; - } - - } - - public LuceneDatabaseLoadTestCase() { - super(); - initializeStrings(); - } - - public LuceneDatabaseLoadTestCase(String name) { - super(name); - initializeStrings(); - } - - @Override - protected void setUp() throws Exception { - super.setUp(); - - database = new LuceneDatabase(); - // database.open(); - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - database.close(); - } - - public void testMassInsert() throws Exception { - database.open(true); - - long contentSize = 0; - - int c = 25000; - Date begin = new Date(); - - StringReader reader = null; - List<Reader> readers = new ArrayList<Reader>(); - - for (int i = 0; i < c; i++) { - MetaData[] mes = mdg.generateVersionnedMetaDataEntity(); - for (int j = 0; j < mes.length; j++) { - - int wordcount = 10000 + r.nextInt(20000); - - StringBuffer sb = new StringBuffer(); - for (int k = 0; k < wordcount; k++) { - sb.append(randomstrings[r.nextInt(cs)]).append(" "); - } - - String buf = sb.toString(); - contentSize += buf.length(); - reader = new StringReader(buf); - readers.clear(); - readers.add(reader); - - database.insertElement(mes[j], readers); - } - if (i > 0 && i % 100 == 0) { - - database.commit(); - - Date end = new Date(); - long time = end.getTime() - begin.getTime(); - System.out.println("Insert " + i + " : Time taken : " + time - + "ms"); - double timePerElement = time / (4 * i); - System.out.println("Insert " + i - + " : Time taken per element : " + timePerElement - + "ms"); - - System.out.println("Content size : " + contentSize); - } - - } - database.commit(); - - Date end = new Date(); - long time = end.getTime() - begin.getTime(); - System.out.println("Insert : Time taken : " + time + "ms"); - double timePerElement = time / (4 * c); - System.out.println("Insert : Time taken per element : " - + timePerElement + "ms"); - System.out.println("Content size : " + contentSize); - } - - public void testMassSearchPaginated() throws Exception { - database.open(); - - Date begin = new Date(); - - int csearch = 100; - - for (int i = 0; i < csearch; i++) { - String key = randomstrings[r.nextInt(randomstrings.length)] + "*"; - - int count = database.findElementsByContentSearchCount(key, false); - System.out.println(key + " : " + count); - - int start = count / 2; - int length = Math.min(200, Math.min(count, Math.max(1, count / 4))); - - Set<MetaData> metaDatas = database - .findElementsByContentSearch(key, false, start, length, 1); - - System.out.println(key + " (" + start + " " + length + ") : " - + metaDatas.size()); - - } - Date end = new Date(); - long time = end.getTime() - begin.getTime(); - System.out.println("Search : Time taken : " + time + "ms"); - double timePerElement = time / csearch; - System.out.println("Search : Time taken per element : " - + timePerElement + "ms"); - - } -} Deleted: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java 2008-02-02 15:04:22 UTC (rev 579) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseTestCase.java 2008-02-02 15:47:56 UTC (rev 580) @@ -1,139 +0,0 @@ -/* -* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -* ##% */ -package fr.cemagref.simexplorer.is.storage.test; - -import java.io.Reader; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Random; -import java.util.Set; - -import junit.framework.TestCase; -import fr.cemagref.simexplorer.is.entities.metadata.MetaData; -import fr.cemagref.simexplorer.is.entities.metadata.Version; -import fr.cemagref.simexplorer.is.storage.database.Database; -import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; -import fr.cemagref.simexplorer.is.storage.MetaDataGenerator; - -public class LuceneDatabaseTestCase extends TestCase { - - private Database database; - // private ElementGenerator elementGenerator; - private MetaDataGenerator mdg = new MetaDataGenerator(); - private Random r = new Random(); - - @Override - protected void setUp() throws Exception { - super.setUp(); - - database = new LuceneDatabase(); - database.open(true); - // elementGenerator = new ElementGenerator(); - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - database.close(); - } - - public void testInsertElementGetElement() throws Exception { - MetaData me = mdg.generateMetaDataEntity(); - database.insertElement(me, new ArrayList<Reader>()); - database.commit(); - MetaData me2 = database.getElement(me.getUuid(), me.getVersion()); - assertEquals(me.getDescription(), me2.getDescription()); - database.insertElement(me, new ArrayList<Reader>()); - database.commit(); - } - - public void testIndexAddFindElementsByContentSearch() throws Exception { - MetaData me = mdg.generateMetaDataEntity(); - - String keyphrase = Long.toHexString(r.nextLong()); - - StringReader reader = new StringReader("ogijeriog ergijergioj " - + keyphrase + " egiojergij ergijerogij"); - - List<Reader> readers = new ArrayList<Reader>(); - readers.add(reader); - - database.insertElement(me, readers); - database.commit(); - - int mesCount = database.findElementsByContentSearchCount(keyphrase, - true); - - assertEquals(1, mesCount); - } - - public void testGetVersions() throws Exception { - MetaData[] mes = mdg.generateVersionnedMetaDataEntity(); - for (int i = 0; i < mes.length; i++) { - database.insertElement(mes[i], new ArrayList<Reader>()); - } - database.commit(); - - List<Version> versions = database.getVersions(mes[0].getUuid()); - assertEquals(mes.length, versions.size()); - Collections.sort(versions); - assertEquals("1.0", versions.get(0).toString()); - assertEquals("1.1", versions.get(1).toString()); - assertEquals("1.2", versions.get(2).toString()); - assertEquals("2.0", versions.get(3).toString()); - } - - public void testFindElementsById() throws Exception { - MetaData[] mes = mdg.generateVersionnedMetaDataEntity(); - for (int i = 0; i < mes.length; i++) { - database.insertElement(mes[i], new ArrayList<Reader>()); - } - database.commit(); - - Set<MetaData> elements = database.findElementsById(mes[0] - .getUuid(), false); - assertEquals(elements.size(), mes.length); - } - - public void testFindElementsByProperties() throws Exception { - MetaData me = mdg.generateMetaDataEntity(); - - database.insertElement(me, new ArrayList<Reader>()); - database.commit(); - - } - - public void testDeleteElements() { - fail("Not yet implemented"); - } - - public void testDeleteElementStringVersion() { - fail("Not yet implemented"); - } - - public void testGetElementLatestVersion() { - fail("Not yet implemented"); - } - - public void testDeleteElementMetaDataEntity() { - fail("Not yet implemented"); - } - -} Deleted: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java 2008-02-02 15:04:22 UTC (rev 579) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThread.java 2008-02-02 15:47:56 UTC (rev 580) @@ -1,114 +0,0 @@ -/* -* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -* ##% */ -package fr.cemagref.simexplorer.is.storage.test; - -import java.io.Reader; -import java.util.ArrayList; -import java.util.Random; -import java.util.Set; - -import fr.cemagref.simexplorer.is.entities.metadata.MetaData; -import fr.cemagref.simexplorer.is.storage.database.Database; -import fr.cemagref.simexplorer.is.storage.MetaDataGenerator; - -public class LuceneDatabaseThread extends Thread { - - private Database database; - private int idThreadLucene; - - private MetaDataGenerator mdg = new MetaDataGenerator(); - - /** - * @return the database - */ - public Database getDatabase() { - return database; - } - - /** - * @param database - * the database to set - */ - public void setDatabase(Database database) { - this.database = database; - } - - @Override - public void run() { - int npass = 10; - String[] ids = new String[npass]; - for (int i = 0; i < npass; i++) { - try { - ids[i] = insertElements(i); - } catch (Exception e) { - System.err.println(idThreadLucene + " - insert : " + e); - } - } - - for (int i = 0; i < npass; i++) { - try { - searchElements(ids[i]); - } catch (Exception e) { - System.err.println(idThreadLucene + " - search : " + e); - } - } - - } - - private void searchElements(String id) throws Exception { - Set<MetaData> mde = database.findElementsById(id, true); - if (mde.isEmpty()) { - System.err.println(idThreadLucene + " - " + id); - } - } - - private String insertElements(int ipass) throws Exception { - - String id = null; - - int ncycles = 5 + (new Random()).nextInt(20); - - for (int j = 0; j < ncycles; j++) { - MetaData[] mes = mdg.generateVersionnedMetaDataEntity(); - for (int i = 0; i < mes.length; i++) { - mes[i].setParentDataUuid(Integer.toString(idThreadLucene)); - database.insertElement(mes[i], new ArrayList<Reader>()); - id = mes[i].getUuid(); - } - } - System.out.print(idThreadLucene + " "); - database.commit(); - return id; - } - - /** - * @return the idThreadLucene - */ - public int getIdThreadLucene() { - return idThreadLucene; - } - - /** - * @param idThreadLucene - * the idThreadLucene to set - */ - public void setIdThreadLucene(int idThreadLucene) { - this.idThreadLucene = idThreadLucene; - } - -} Deleted: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java 2008-02-02 15:04:22 UTC (rev 579) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneDatabaseThreadsTestCase.java 2008-02-02 15:47:56 UTC (rev 580) @@ -1,59 +0,0 @@ -/* -* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -* ##% */ -package fr.cemagref.simexplorer.is.storage.test; - -import junit.framework.TestCase; -import fr.cemagref.simexplorer.is.storage.database.Database; -import fr.cemagref.simexplorer.is.storage.database.lucene.LuceneDatabase; - -public class LuceneDatabaseThreadsTestCase extends TestCase { - - private Database database; - - @Override - protected void setUp() throws Exception { - super.setUp(); - database = new LuceneDatabase(); - database.open(true); - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - database.close(); - } - - public void testLuceneDatabaseThreads() throws Exception { - - int nthreads = 10; - - LuceneDatabaseThread[] threads = new LuceneDatabaseThread[nthreads]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new LuceneDatabaseThread(); - threads[i].setDatabase(database); - threads[i].setIdThreadLucene(i); - threads[i].start(); - } - - for (int i = 0; i < threads.length; i++) { - threads[i].join(); - } - - } - -} Deleted: trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java =================================================================== --- trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java 2008-02-02 15:04:22 UTC (rev 579) +++ trunk/simexplorer-is-storage/src/test/fr/cemagref/simexplorer/is/storage/test/LuceneSimpleTest.java 2008-02-02 15:47:56 UTC (rev 580) @@ -1,89 +0,0 @@ -/* -* ##% Copyright (C) 2008 Code Lutin, Gabriel Landais -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -* ##% */ -package fr.cemagref.simexplorer.is.storage.test; - -import junit.framework.TestCase; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.SimpleAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Searcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; - -public class LuceneSimpleTest extends TestCase { - - /* - * public void testSearch() throws Exception { Searcher searcher; searcher = - * new IndexSearcher("/home/landais/lucene/index.big"); Query query = new - * TermQuery(new Term("simexplorer.type", "LuceneDatabaseTestCase")); - * - * Sort sort = new Sort("simexplorer.creationdate", true); Hits hits = - * searcher.search(query, sort); for (int i = 0; i < 50; i++) { Document doc = - * hits.doc(i); System.out.println(doc.get("simexplorer.creationdate")); } - * System.out.println(hits.length()); } - */ - - public void testLuceneSimple() throws Exception { - - IndexWriter writer; - Searcher searcher; - Analyzer analyser; - Query query; - - /* - * String indexDir = "./index/"; searcher = new IndexSearcher(indexDir); - * query = new TermQuery(new Term("simexplorer.type", - * "LuceneDatabaseTestCase")); Filter filter = new QueryFilter(query); - * System.out.println(searcher.search(query, null, 10).totalHits); - */ - - String indexDir = "./indextest/"; - analyser = new SimpleAnalyzer(); - Directory dir = FSDirectory.getDirectory(indexDir); - writer = new IndexWriter(dir, true, analyser, true); - - searcher = new IndexSearcher(indexDir); - - Document doc = new Document(); - - doc.add(new Field("field1", "abcd", Field.Store.YES, - Field.Index.UN_TOKENIZED)); - doc.add(new Field("field2", "dcba", Field.Store.YES, - Field.Index.UN_TOKENIZED)); - - writer.addDocument(doc); - - query = new TermQuery(new Term("field1", "abcd")); - - System.out.println(searcher.search(query).length()); - writer.flush(); - System.out.println(searcher.search(query).length()); - writer.close(); - System.out.println(searcher.search(query).length()); - searcher = new IndexSearcher(indexDir); - System.out.println(searcher.search(query).length()); - } - -}