Author: bpoussin Date: 2010-12-20 19:21:26 +0100 (Mon, 20 Dec 2010) New Revision: 636 Url: http://nuiton.org/repositories/revision/wikitty/636 Log: - change maxBooleanClauses to Integer.MAX_VALUE - cahnge copyField position after fields declaration (view in documentation) - change restoreNode restoreChild to restrict filter resultat to attachment id - add test (that faild :() on word indexation with accent :( Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java trunk/wikitty-solr-impl/src/main/resources/schema.xml trunk/wikitty-solr-impl/src/main/resources/solrconfig.xml trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/WikittyServiceSolr.java Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java 2010-12-20 18:18:00 UTC (rev 635) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/Restriction2Solr.java 2010-12-20 18:21:26 UTC (rev 636) @@ -172,6 +172,8 @@ } } + // FIXME poussin 20101220 i think this method return bad lucene syntaxe query + // I think the syntaxe must be the same as contains private String in2solr(In in) { boolean first = true; String result = in.getElement().getName() + ":["; @@ -381,11 +383,11 @@ String operand = ""; StringBuffer result = new StringBuffer(); + result.append(element2solr(contains.getElement())).append(":"); result.append("("); for (String value : contains.getValue()) { result.append(operand); - result.append(element2solr(contains.getElement())) - .append(":").append(value2solr(value)); + result.append(value2solr(value)); operand = " OR "; } result.append(")"); Modified: trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java =================================================================== --- trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java 2010-12-20 18:18:00 UTC (rev 635) +++ trunk/wikitty-solr-impl/src/main/java/org/nuiton/wikitty/storage/solr/WikittySearchEngineSolrIndexInTreeNode.java 2010-12-20 18:21:26 UTC (rev 636) @@ -65,9 +65,12 @@ import java.util.LinkedHashSet; import java.util.LinkedList; import org.nuiton.util.ApplicationConfig; +import org.nuiton.util.CollectionUtil; +import org.nuiton.util.StringUtil; import org.nuiton.wikitty.WikittyConfig; import org.nuiton.wikitty.WikittyUtil; import org.nuiton.wikitty.entities.WikittyTreeNodeHelper; +import org.nuiton.wikitty.search.Search; /** * @@ -514,6 +517,10 @@ } } if (filter != null) { + Search search = Search.query(filter); + Search and = search.and().contains(SOLR_ID, allAttachments); + Criteria criteria = search.criteria(); + PagedResult<String> criteriaResult = findAllByCriteria(transaction, filter); allAttachments.retainAll(criteriaResult.getAll()); } @@ -588,13 +595,38 @@ // creation des id repondants au filtre si besoin Collection<String> filteredId = Collections.EMPTY_LIST; if (filter != null) { - PagedResult<String> criteriaResult = findAllByCriteria(transaction, filter); - filteredId = criteriaResult.getAll(); + + // stocke toutes les ids des attachments du sous arbres + Collection<String> attachmentIds = new LinkedList<String>(); + // 1ere passe, on ne garde que les fils directs. + // comme ca se sera deja fait pour la construction du result + // moins de chose a reparcourir + for (Iterator<Map.Entry<String, Collection>> i = allAttachments.entrySet().iterator(); i.hasNext();) { + Map.Entry<String, Collection> e = i.next(); + // on ne garde que les fils directs + if (child.contains(e.getKey())) { + attachmentIds.addAll(e.getValue()); + } else { + i.remove(); + } + } + + // ca ne sert a rien de faire la requete si au final, il n'y a + // aucun attachment + if (attachmentIds.size() > 0) { + Search search = Search.query(filter); + Search and = search.and().contains(SOLR_ID, attachmentIds); + Criteria criteria = search.criteria(); + + PagedResult<String> criteriaResult = findAllByCriteria(transaction, criteria); + filteredId = criteriaResult.getAll(); + } } // Construction du resultat for (Map.Entry<String, Collection> e : allAttachments.entrySet()) { // on ne garde que les fils directs + // c'est peut-etre fait si filter est non null, mais pas sinon if (child.contains(e.getKey())) { Collection attachments = e.getValue(); // on ne garde dans le comptage des attachments que les id Modified: trunk/wikitty-solr-impl/src/main/resources/schema.xml =================================================================== --- trunk/wikitty-solr-impl/src/main/resources/schema.xml 2010-12-20 18:18:00 UTC (rev 635) +++ trunk/wikitty-solr-impl/src/main/resources/schema.xml 2010-12-20 18:21:26 UTC (rev 636) @@ -48,12 +48,16 @@ <fieldType name="date" class="solr.DateField" sortMissingLast="true" omitNorms="true"/> <!-- STRING type: The StrField type is not analyzed, but indexed/stored verbatim. --> - <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="string" class="solr.StrField" + compressed="true" compressThreshold="1000" + sortMissingLast="true" omitNorms="true"/> <!-- STRING type copy: type to string all text is lower cased --> - <fieldType name="string_lc" class="solr.StrField" sortMissingLast="true" omitNorms="true"> + <fieldType name="string_lc" class="solr.StrField" + compressed="true" compressThreshold="1000" + sortMissingLast="true"> <analyzer> <!-- no type to indicated that used it for both type: index and query --> - <tokenizer class="solr.StandardTokenizerFactory"/> <!-- ou ? WhitespaceTokenizerFactory --> + <tokenizer class="solr.StandardTokenizerFactory"/> <filter class="solr.LowerCaseFilterFactory"/> </analyzer> </fieldType> @@ -66,7 +70,9 @@ Duplicate tokens at the same position (which may result from Stemmed Synonyms or WordDelim parts) are removed. --> - <fieldType name="text" class="solr.TextField" positionIncrementGap="100"> + <fieldType name="text" class="solr.TextField" + compressed="true" compressThreshold="1000" + positionIncrementGap="100"> <analyzer type="index"> <tokenizer class="solr.WhitespaceTokenizerFactory"/> <filter class="solr.StopFilterFactory" @@ -119,21 +125,10 @@ <dynamicfield name="#tree*" type="string" indexed="true" stored="false" multiValued="true"/> <!-- copy all field (except binary) in '#fulltext' field for fulltext search --> - <field name="#fulltext" type="text" indexed="true" stored="false" multiValued="true"/> - <copyField source="*_b" dest="#fulltext"/> - <copyField source="*_d" dest="#fulltext"/> - <copyField source="*_dt" dest="#fulltext"/> - <copyField source="*_s" dest="#fulltext"/> - <copyField source="*_w" dest="#fulltext"/> - - <!-- copy String field for to lower case version --> - <copyField source="*_s" dest="*_s_c"/> - <!-- copy String field for text indexed format version --> - <copyField source="*_s" dest="*_s_t"/> - + <field name="#fulltext" type="text" indexed="true" stored="true" multiValued="true"/> <!-- copied field not stored --> - <dynamicField name="*_s_c" type="string_lc" indexed="true" stored="false" multiValued="true"/> - <dynamicField name="*_s_t" type="text" indexed="true" stored="false" multiValued="true"/> + <dynamicField name="*_s_c" type="string_lc" indexed="true" stored="true" multiValued="true"/> + <dynamicField name="*_s_t" type="text" indexed="true" stored="true" multiValued="true"/> <!-- on indexe pas les binary field --> <dynamicField name="*_bi" type="binary" indexed="false" stored="false" multiValued="true"/> @@ -160,4 +155,17 @@ <!-- SolrQueryParser configuration: defaultOperator="AND|OR" --> <solrQueryParser defaultOperator="AND"/> + + <copyField source="*_b" dest="#fulltext"/> + <copyField source="*_d" dest="#fulltext"/> + <copyField source="*_dt" dest="#fulltext"/> + <copyField source="*_s" dest="#fulltext"/> + <copyField source="*_w" dest="#fulltext"/> + + <!-- copy String field for to lower case version --> + <copyField source="*_s" dest="*_s_c"/> + <!-- copy String field for text indexed format version --> + <copyField source="*_s" dest="*_s_t"/> + + </schema> Modified: trunk/wikitty-solr-impl/src/main/resources/solrconfig.xml =================================================================== --- trunk/wikitty-solr-impl/src/main/resources/solrconfig.xml 2010-12-20 18:18:00 UTC (rev 635) +++ trunk/wikitty-solr-impl/src/main/resources/solrconfig.xml 2010-12-20 18:21:26 UTC (rev 636) @@ -124,10 +124,10 @@ <query> - <!-- Maximum number of clauses in a boolean query... can affect + <!-- Maximum number of clauses in a boolean query (default: 1024). can affect range or prefix queries that expand to big boolean queries. An exception is thrown if exceeded. --> - <maxBooleanClauses>1024</maxBooleanClauses> + <maxBooleanClauses>2147483647</maxBooleanClauses> <!-- Cache used by SolrIndexSearcher for filters (DocSets), Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java =================================================================== --- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java 2010-12-20 18:18:00 UTC (rev 635) +++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/SolrSearchTest.java 2010-12-20 18:21:26 UTC (rev 636) @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.List; +import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -45,6 +46,8 @@ import org.nuiton.wikitty.WikittyUtil; import org.nuiton.wikitty.conform.AbstractTestConformance; import org.nuiton.wikitty.conform.StorageTest; +import org.nuiton.wikitty.entities.WikittyGroup; +import org.nuiton.wikitty.entities.WikittyGroupImpl; import org.nuiton.wikitty.search.operators.Element; import org.nuiton.wikitty.search.operators.Like; import org.nuiton.wikitty.search.operators.Like.SearchAs; @@ -504,4 +507,42 @@ list = result.getAll(); assertEquals(2, list.size()); } + + @Test + public void testAccent() throws Exception { + WikittyGroupImpl group = new WikittyGroupImpl(); + group.setName("coucou"); + WikittyGroupImpl groupAccent = new WikittyGroupImpl(); + groupAccent.setName("çéçà"); + + ws.store(null, group.getWikitty()); + ws.store(null, groupAccent.getWikitty()); + + { + // sans accent + Criteria criteria = Search.query().keyword("coucou").criteria(); + PagedResult<String> result = ws.findAllByCriteria(null, criteria); + Assert.assertEquals(1, result.size()); + } + { + // avec accent + Criteria criteria = Search.query().keyword("çéçà").criteria(); + PagedResult<String> result = ws.findAllByCriteria(null, criteria); + Assert.assertEquals(1, result.size()); + } + { + // avec accent specifiquement sur le champs name + Criteria criteria = Search.query().eq( + WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME,"çéçà").criteria(); + PagedResult<String> result = ws.findAllByCriteria(null, criteria); + Assert.assertEquals(1, result.size()); + } + { + // avec accent specifiquement sur le champs name mais sans accent pour la recherche + Criteria criteria = Search.query().eq( + WikittyGroup.FQ_FIELD_WIKITTYGROUP_NAME,"ceca").criteria(); + PagedResult<String> result = ws.findAllByCriteria(null, criteria); + Assert.assertEquals(1, result.size()); + } + } } Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java =================================================================== --- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java 2010-12-20 18:18:00 UTC (rev 635) +++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/TreeTest.java 2010-12-20 18:21:26 UTC (rev 636) @@ -30,6 +30,7 @@ import static junit.framework.Assert.assertNotNull; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -265,6 +266,11 @@ @Test public void testFilterRestoreChildren() { + Map<String, Integer> result = new HashMap<String, Integer>(); + result.put("node11", 1); + result.put("node13", 0); + result.put("node12", 0); + Wikitty node1 = findNode("node1"); String node1Id = node1.getId(); @@ -272,6 +278,9 @@ Map<WikittyTreeNode, Integer> children = ws.restoreChildren(null, node1Id, filter); System.out.println(children); assertEquals(3, children.size()); + for (Map.Entry<WikittyTreeNode, Integer> e : children.entrySet()) { + assertEquals(result.get(e.getKey().getName()), e.getValue()); + } } @Test Modified: trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/WikittyServiceSolr.java =================================================================== --- trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/WikittyServiceSolr.java 2010-12-20 18:18:00 UTC (rev 635) +++ trunk/wikitty-solr-impl/src/test/java/org/nuiton/wikitty/storage/solr/WikittyServiceSolr.java 2010-12-20 18:21:26 UTC (rev 636) @@ -39,7 +39,6 @@ * by : $Author$ */ public class WikittyServiceSolr extends WikittyServiceStorage { - public WikittyServiceSolr(ApplicationConfig config) { extensionStorage = new WikittyExtensionStorageInMemory(); wikittyStorage = new WikittyStorageInMemory();