[Chorem-commits] r264 - in trunk: chorem-entities/src/main/java/org/chorem chorem-entities/src/main/xmi chorem-webmotion/src/main/java/org/chorem/webmotion/actions

Author: bpoussin Date: 2012-09-13 12:09:57 +0200 (Thu, 13 Sep 2012) New Revision: 264 Url: http://chorem.org/repositories/revision/chorem/264 Log: - une facture peut avoir comme supplier/customer: Company, Employee ou Person - ajout de l'import des factures et cout depuis chorem-topia Modified: trunk/chorem-entities/src/main/java/org/chorem/ChoremAction.java trunk/chorem-entities/src/main/java/org/chorem/ChoremConfigAction.java trunk/chorem-entities/src/main/java/org/chorem/ChoremMain.java trunk/chorem-entities/src/main/java/org/chorem/ImportChoremTopia.java trunk/chorem-entities/src/main/xmi/chorem-model.properties trunk/chorem-entities/src/main/xmi/chorem-model.zargo trunk/chorem-webmotion/src/main/java/org/chorem/webmotion/actions/DashboardAction.java Modified: trunk/chorem-entities/src/main/java/org/chorem/ChoremAction.java =================================================================== --- trunk/chorem-entities/src/main/java/org/chorem/ChoremAction.java 2012-09-10 06:43:45 UTC (rev 263) +++ trunk/chorem-entities/src/main/java/org/chorem/ChoremAction.java 2012-09-13 10:09:57 UTC (rev 264) @@ -25,6 +25,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.ApplicationConfig; +import org.nuiton.wikitty.query.WikittyQuery; +import org.nuiton.wikitty.query.WikittyQueryMaker; +import org.nuiton.wikitty.query.WikittyQueryResult; /** * @@ -46,15 +49,42 @@ } /** - * Remove all data in database, used with caution + * Vide les objets qui porte une certaine extensions + * @param extensionName */ - public void clean() { - System.out.println("Cleaning database ..."); + @ApplicationConfig.Action.Step(0) + public void clean(String extensionName) { + System.out.println("##### EFFACEMENT: " + extensionName + " #####"); ChoremClient proxy = ChoremClient.getClient(config); - proxy.clear(); + WikittyQuery q = new WikittyQueryMaker().exteq(extensionName) + .end().setLimit(WikittyQuery.MAX); + WikittyQueryResult<String> result = proxy.findAllByQuery(q); + proxy.delete(result.getAll()); System.out.println("... clean done"); } + /** + * Vide tous les objets + */ + @ApplicationConfig.Action.Step(0) + public void clear() { + System.out.println("##### EFFACEMENT TOTAL DES DONNEES #####"); + ChoremClient proxy = ChoremClient.getClient(config); + proxy.clear(); + System.out.println("... clear done"); + } + + /** + * Reindex les donnees existantes + */ + @ApplicationConfig.Action.Step(0) + public void reindex() { + System.out.println("##### REINDEXATION #####"); + ChoremClient proxy = ChoremClient.getClient(config); + proxy.syncSearchEngine(); + System.out.println("... reindexation done"); + } + public void configInfo() { config.printConfig(); } Modified: trunk/chorem-entities/src/main/java/org/chorem/ChoremConfigAction.java =================================================================== --- trunk/chorem-entities/src/main/java/org/chorem/ChoremConfigAction.java 2012-09-10 06:43:45 UTC (rev 263) +++ trunk/chorem-entities/src/main/java/org/chorem/ChoremConfigAction.java 2012-09-13 10:09:57 UTC (rev 264) @@ -37,6 +37,8 @@ HELP(ChoremAction.class.getName() + "#help", "-h", "--help"), INFO(ChoremAction.class.getName() + "#configInfo", "-i", "--config-info" , "--configInfo"), CLEAN(ChoremAction.class.getName() + "#clean", "--clean"), + CLEAR(ChoremAction.class.getName() + "#clear", "--clear"), + REINDEX(ChoremAction.class.getName() + "#reindex", "--reindex"), IMPORT_PERSON(ImportChoremTopia.class.getName() + "#importPerson", "-ip", "--import-person", "--importPerson"), IMPORT_COMPANY(ImportChoremTopia.class.getName() + "#importCompany", @@ -47,6 +49,10 @@ "--import-contract", "--importContract"), IMPORT_CONTRACT_TYPE(ImportChoremTopia.class.getName() + "#importContractType", "--import-contract-type", "--importContractType"), + IMPORT_INVOICE(ImportChoremTopia.class.getName() + "#importInvoices", + "-ii", "--import-invoices", "--importInvoices"), + IMPORT_COST(ImportChoremTopia.class.getName() + "#importCosts", + "--import-costs", "--importCosts"), IMPORT_COMMIT(ImportChoremTopia.class.getName() + "#commit", "--commit"); public String action; Modified: trunk/chorem-entities/src/main/java/org/chorem/ChoremMain.java =================================================================== --- trunk/chorem-entities/src/main/java/org/chorem/ChoremMain.java 2012-09-10 06:43:45 UTC (rev 263) +++ trunk/chorem-entities/src/main/java/org/chorem/ChoremMain.java 2012-09-13 10:09:57 UTC (rev 264) @@ -41,9 +41,11 @@ static private Log log = LogFactory.getLog(ChoremMain.class); static public void main(String[] args) throws Exception { +// args = "--clear -ic /tmp/chorem-company.csv -ip /tmp/chorem-person.csv -ie /tmp/chorem-employee.csv --import-contract-type /tmp/chorem-contracttype.csv --import-contract /tmp/chorem-contract.csv --commit".split(" "); +// args = "-ii /tmp/bill.csv --import-costs /tmp/cost.csv --commit".split(" "); + args = "--reindex".split(" "); System.out.println(String.format("Launching ChoremMain ... (args: %s)", Arrays.toString(args))); // getConfig do all: parse and doAllAction -// String[] forceArgs = "--clean -ic /tmp/chorem-company.csv -ip /tmp/chorem-person.csv -ie /tmp/chorem-employee.csv --import-contract-type /tmp/chorem-contracttype.csv --import-contract /tmp/chorem-contract.csv --commit".split(" "); ApplicationConfig config = ChoremConfig.getConfig(null, args); System.exit(0); Modified: trunk/chorem-entities/src/main/java/org/chorem/ImportChoremTopia.java =================================================================== --- trunk/chorem-entities/src/main/java/org/chorem/ImportChoremTopia.java 2012-09-10 06:43:45 UTC (rev 263) +++ trunk/chorem-entities/src/main/java/org/chorem/ImportChoremTopia.java 2012-09-13 10:09:57 UTC (rev 264) @@ -27,20 +27,32 @@ import java.text.ParseException; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.chorem.entities.Category; +import org.chorem.entities.CategoryImpl; +import org.chorem.entities.Company; import org.chorem.entities.CompanyImpl; import org.chorem.entities.ContactDetailsImpl; +import org.chorem.entities.Employee; import org.chorem.entities.EmployeeImpl; +import org.chorem.entities.Invoice; import org.chorem.entities.InvoiceImpl; +import org.chorem.entities.Person; import org.chorem.entities.PersonImpl; import org.nuiton.util.ApplicationConfig; import org.nuiton.util.ApplicationConfig.Action.Step; import org.nuiton.wikitty.entities.BusinessEntityImpl; +import org.nuiton.wikitty.query.WikittyQuery; +import org.nuiton.wikitty.query.WikittyQueryMaker; +import org.nuiton.wikitty.query.WikittyQueryResult; /** * Les donnees doivent etre exportees de la base postgresql avec les commandes: @@ -53,6 +65,7 @@ * * pour bill * \copy (SELECT subcategory.name AS category, number, issuedate, hopedate, realdate, value, company.name AS company from bill LEFT OUTER JOIN subcategory ON bill.subcategory=subcategory.topiaid LEFT OUTER JOIN task ON bill.task=task.topiaid LEFT OUTER JOIN contact ON bill.contact=contact.topiaid LEFT OUTER JOIN company ON contact.company=company.topiaid order by company.name) TO '/tmp/bill.csv' delimiter ';' CSV HEADER + * \copy (SELECT date,subcategory.name AS category,tag,value from tagvalue LEFT OUTER JOIN subcategory ON tagvalue.subcategory=subcategory.topiaid) TO '/tmp/cost.csv' delimiter ';' CSV HEADER * </pre> * * @author poussin @@ -99,11 +112,17 @@ category,number,issuedate,hopedate,realdate,value,company } + enum COST_HEADER { + date,category,tag,value + } + protected List<ContactDetailsImpl> contacts = new LinkedList<ContactDetailsImpl>(); protected Map<String, CompanyImpl> companies = new HashMap<String, CompanyImpl>(); protected Map<String, PersonImpl> persons = new HashMap<String, PersonImpl>(); protected Map<String, EmployeeImpl> employees = new HashMap<String, EmployeeImpl>(); protected Map<String, InvoiceImpl> invoices = new HashMap<String, InvoiceImpl>(); + protected Map<String, InvoiceImpl> costs = new HashMap<String, InvoiceImpl>(); + protected Map<String, CategoryImpl> categories = new HashMap<String, CategoryImpl>(); // a ne pas stocker dans wikitty protected Map<String, String> contractType = new HashMap<String, String>(); @@ -149,11 +168,14 @@ @Step(99) public void commit() { List<BusinessEntityImpl> data = new LinkedList<BusinessEntityImpl>(); - + + data.addAll(categories.values()); data.addAll(companies.values()); data.addAll(persons.values()); data.addAll(employees.values()); data.addAll(contacts); + data.addAll(invoices.values()); + data.addAll(costs.values()); ChoremClient proxy = ChoremClient.getClient(config); data = proxy.store(data); @@ -429,4 +451,230 @@ System.out.println(String.format("%s contracts loaded", count)); } + @Step(4) + public void importInvoices(String filename) { + System.out.println(String.format("Try to import invoices '%s'", filename)); + try { + ChoremClient proxy = ChoremClient.getClient(config); + + // Pour les invoices existantes + WikittyQuery invoicesQuery = new WikittyQueryMaker() + .exteq(Invoice.EXT_INVOICE).end().setLimit(WikittyQuery.MAX); + WikittyQueryResult<Invoice> invoicesResult = + proxy.findAllByQuery(Invoice.class, invoicesQuery); + Set<String> knowInvoices = new HashSet<String>(); + Set<String> passedInvoices = new HashSet<String>(); + Set<String> rejectedInvoices = new HashSet<String>(); + Set<String> treatedInvoices = new HashSet<String>(); + for (Invoice c : invoicesResult) { + knowInvoices.add(c.getReference()); + } + + // Pour les categories + WikittyQuery categoryQuery = new WikittyQueryMaker() + .exteq(Category.EXT_CATEGORY).end().setLimit(WikittyQuery.MAX); + WikittyQueryResult<Category> categoryResult = + proxy.findAllByQuery(Category.class, categoryQuery); + // key: category name; value: id + Map<String, String> categories = new HashMap<String, String>(); + Set<String> unknowCategories = new HashSet<String>(); + for (Category c : categoryResult) { + categories.put(c.getName(), c.getWikittyId()); + } + + // Pour les customer + WikittyQuery customerQuery = new WikittyQueryMaker() + .exteq(Company.EXT_COMPANY) + .end().setLimit(WikittyQuery.MAX); + WikittyQueryResult<Company> customerResult = + proxy.findAllByQuery(Company.class, customerQuery); + // key: name; value: id + Map<String, String> customers = new HashMap<String, String>(); + Set<String> unknowCustomers = new HashSet<String>(); + for (Company c : customerResult) { + customers.put(c.getName(), c.getWikittyId()); + } + + // Pour les supplier + WikittyQuery supplierQuery = new WikittyQueryMaker() + .eq(Company.FQ_FIELD_COMPANY_NAME, "Code Lutin").end(); + String codeLutinId = proxy.findByQuery(supplierQuery); + + FileReader in = new FileReader(filename); + // separateur ';' String delimiter '"', skip first line (header) + CSVReader reader = new CSVReader(in, ';', '"', 1); + for(String[] line : reader.readAll()) { + String number = line[BILL_HEADER.number.ordinal()]; + if (knowInvoices.contains(number)) { + passedInvoices.add(number); + } else { + String category = line[BILL_HEADER.category.ordinal()]; + String company = line[BILL_HEADER.company.ordinal()]; + if (codeLutinId == null + || !categories.containsKey(category) + || !customers.containsKey(company)) { + rejectedInvoices.add(number); + // check des infos + if (!categories.containsKey(category)) { + unknowCategories.add(category); + } + + if (!customers.containsKey(company)) { + unknowCustomers.add(company); + } + } else { + Double amount = Double.parseDouble(line[BILL_HEADER.value.ordinal()])/100.0; + Date issuedate = parseDate(line[BILL_HEADER.issuedate.ordinal()]); + Date hopedate = parseDate(line[BILL_HEADER.hopedate.ordinal()]); + Date realdate = parseDate(line[BILL_HEADER.realdate.ordinal()]); + + InvoiceImpl invoice = new InvoiceImpl(); + invoice.setCategory(categories.get(category)); + invoice.setSupplier(codeLutinId); + invoice.setCustomer(company); + invoice.setReference(number); + invoice.setAmount(amount); + invoice.setPostedDate(issuedate); + invoice.setExpectedDate(hopedate); + invoice.setPaymentDate(realdate); + + invoices.put(invoice.getWikittyId(), invoice); + treatedInvoices.add(number); + } + } + } + + reader.close(); + + System.out.println("Invoices deja importee: " + passedInvoices); + System.out.println("Invoices a importee: " + treatedInvoices); + if (codeLutinId == null || !unknowCategories.isEmpty() || !unknowCustomers.isEmpty()) { + System.out.println("-- Erreur d'import --"); + System.out.println("Invoices rejetees: " + rejectedInvoices); + System.out.println("Code Lutin id: " + codeLutinId); + System.out.println("Categorie manquante: " + unknowCategories); + System.out.println("Company manquante: " + unknowCustomers); + } + } catch (Exception eee) { + log.fatal(String.format("Can't import file '%s'", filename)); + if (log.isDebugEnabled()) { + log.debug("StackTrace", eee); + } + System.exit(1); + } + System.out.println(String.format("%s invoices loaded", invoices.size())); + } + + protected String invoiceToKey(Invoice c) { + String result = c.getCategory() + ";" + c.getPaymentDate() + ";" + c.getName() + ";" + c.getAmount(); + return result; + } + + @Step(5) + public void importCosts(String filename) { + System.out.println(String.format("Try to import cost '%s'", filename)); + try { + ChoremClient proxy = ChoremClient.getClient(config); + + // Pour les invoices existantes + WikittyQuery invoicesQuery = new WikittyQueryMaker() + .exteq(Invoice.EXT_INVOICE).end().setLimit(WikittyQuery.MAX); + WikittyQueryResult<Invoice> invoicesResult = + proxy.findAllByQuery(Invoice.class, invoicesQuery); + Set<String> knowInvoices = new HashSet<String>(); + Set<String> passedInvoices = new HashSet<String>(); + Set<String> rejectedInvoices = new HashSet<String>(); + Set<String> treatedInvoices = new HashSet<String>(); + for (Invoice c : invoicesResult) { + // on fait une concatenation pour optimiser les futurs recherche + knowInvoices.add(invoiceToKey(c)); + } + + // Pour les categories + WikittyQuery categoryQuery = new WikittyQueryMaker() + .exteq(Category.EXT_CATEGORY).end().setLimit(WikittyQuery.MAX); + WikittyQueryResult<Category> categoryResult = + proxy.findAllByQuery(Category.class, categoryQuery); + // key: category name; value: id + Map<String, String> categoryNameId = new HashMap<String, String>(); + Set<String> unknowCategories = new HashSet<String>(); + for (Category c : categoryResult) { + categoryNameId.put(c.getName(), c.getWikittyId()); + } + + // le customer est code lutin + WikittyQuery customerQuery = new WikittyQueryMaker() + .eq(Company.FQ_FIELD_COMPANY_NAME, "Code Lutin").end(); + String codeLutinId = proxy.findByQuery(customerQuery); + + // la category par defaut ou creer les manquantes + WikittyQuery fraisQuery = new WikittyQueryMaker() + .eq(Category.FQ_FIELD_WIKITTYTREENODE_NAME, "Frais").end(); + String frais = proxy.findByQuery(fraisQuery); + if (frais == null) { + CategoryImpl c = new CategoryImpl(); + c.setName("Frais"); + categories.put(c.getWikittyId(), c); + categoryNameId.put(c.getName(), c.getWikittyId()); + frais = c.getWikittyId(); + } + + FileReader in = new FileReader(filename); + // separateur ';' String delimiter '"', skip first line (header) + CSVReader reader = new CSVReader(in, ';', '"', 1); + for(String[] line : reader.readAll()) { + String tag = line[COST_HEADER.tag.ordinal()]; + String category = line[COST_HEADER.category.ordinal()]; + Date date = parseDate(line[COST_HEADER.date.ordinal()]); + Double amount = Double.parseDouble(line[COST_HEADER.value.ordinal()])/100.0; + + if (StringUtils.isNotBlank(category) && !categoryNameId.containsKey(category)) { + CategoryImpl c = new CategoryImpl(); + c.setName(category); + c.setParent(frais); + categories.put(c.getWikittyId(), c); + categoryNameId.put(c.getName(), c.getWikittyId()); + unknowCategories.add(category); + } + + InvoiceImpl invoice = new InvoiceImpl(); + invoice.setName(tag); + invoice.setCategory(categoryNameId.get(category)); + invoice.setCustomer(codeLutinId); + invoice.setAmount(amount); + invoice.setPostedDate(date); + invoice.setExpectedDate(date); + invoice.setPaymentDate(date); + + String invoiceKey = invoiceToKey(invoice); + if (knowInvoices.contains(invoiceKey)) { + passedInvoices.add(invoiceKey); + } else if (codeLutinId == null || date == null || StringUtils.isBlank(category)) { + rejectedInvoices.add(invoiceKey); + } else { + costs.put(invoice.getWikittyId(), invoice); + treatedInvoices.add(invoiceKey); + } + } + + reader.close(); + + System.out.println("Cost deja importee: " + passedInvoices); + System.out.println("Cost a importee: " + treatedInvoices); + System.out.println("Category creee: " + unknowCategories); + if (codeLutinId == null || !rejectedInvoices.isEmpty()) { + System.out.println("-- Erreur d'import costs--"); + System.out.println("Cost rejetees: " + rejectedInvoices); + System.out.println("Code Lutin id: " + codeLutinId); + } + } catch (Exception eee) { + log.fatal(String.format("Can't import file '%s'", filename)); + if (log.isDebugEnabled()) { + log.debug("StackTrace", eee); + } + System.exit(1); + } + System.out.println(String.format("%s costs loaded", costs.size())); + } + } Modified: trunk/chorem-entities/src/main/xmi/chorem-model.properties =================================================================== --- trunk/chorem-entities/src/main/xmi/chorem-model.properties 2012-09-10 06:43:45 UTC (rev 263) +++ trunk/chorem-entities/src/main/xmi/chorem-model.properties 2012-09-13 10:09:57 UTC (rev 264) @@ -109,7 +109,9 @@ org.chorem.entities.Invoice.attribute.expectedDate.tagvalue.help=La date souhait\u00e9e de paiement org.chorem.entities.Invoice.attribute.paymentDate.tagvalue.help=La date de paiement r\u00e9elle org.chorem.entities.Invoice.attribute.supplier.tagvalue.help=La personne chez Code Lutin qui g\u00e8re la facturation +org.chorem.entities.Invoice.attribute.parent.tagvalue.allowed=Company,Person,Employee org.chorem.entities.Invoice.attribute.customer.tagvalue.help=La personne chez le client \u00e0 qui ont a envoyer la facture +org.chorem.entities.Invoice.attribute.parent.tagvalue.allowed=Company,Person,Employee # # Invoiceable # Modified: trunk/chorem-entities/src/main/xmi/chorem-model.zargo =================================================================== (Binary files differ) Modified: trunk/chorem-webmotion/src/main/java/org/chorem/webmotion/actions/DashboardAction.java =================================================================== --- trunk/chorem-webmotion/src/main/java/org/chorem/webmotion/actions/DashboardAction.java 2012-09-10 06:43:45 UTC (rev 263) +++ trunk/chorem-webmotion/src/main/java/org/chorem/webmotion/actions/DashboardAction.java 2012-09-13 10:09:57 UTC (rev 264) @@ -582,7 +582,7 @@ String title = "Factures à payer par la société"; String companyId = client.getConfiguration().getDefaultCompany(); String filter = Invoice.FQ_FIELD_INVOICE_CUSTOMER + "={SELECT id WHERE (" + - Employee.FQ_FIELD_EMPLOYEE_COMPANY + "=" + companyId+")}"; + Employee.FQ_FIELD_EMPLOYEE_COMPANY + "=" + companyId+" OR " + companyId + ")}"; return invoiceFilter(client, title, filter, start, end, query); } @@ -591,7 +591,7 @@ String title = "Factures émises par la société"; String companyId = client.getConfiguration().getDefaultCompany(); String filter = Invoice.FQ_FIELD_INVOICE_SUPPLIER + "={SELECT id WHERE (" + - Employee.FQ_FIELD_EMPLOYEE_COMPANY + "=" + companyId+")}"; + Employee.FQ_FIELD_EMPLOYEE_COMPANY + "=" + companyId+" OR " + companyId + ")}"; return invoiceFilter(client, title, filter, start, end, query); } @@ -636,7 +636,7 @@ .select(Invoice.FQ_FIELD_INVOICE_AMOUNT, Aggregate.SUM) .and() .containsOne(Invoice.FQ_FIELD_INVOICE_CUSTOMER) - .select(Element.ID).eq(Employee.FQ_FIELD_EMPLOYEE_COMPANY, companyId) + .select(Element.ID).or().ideq(companyId).eq(Employee.FQ_FIELD_EMPLOYEE_COMPANY, companyId).close() .close() .lt(Invoice.FQ_FIELD_INVOICE_EXPECTEDDATE, start) .end() @@ -647,7 +647,7 @@ .select(Invoice.FQ_FIELD_INVOICE_AMOUNT, Aggregate.SUM) .and() .containsOne(Invoice.FQ_FIELD_INVOICE_SUPPLIER) - .select(Element.ID).eq(Employee.FQ_FIELD_EMPLOYEE_COMPANY, companyId) + .select(Element.ID).or().ideq(companyId).eq(Employee.FQ_FIELD_EMPLOYEE_COMPANY, companyId).close() .close() .lt(Invoice.FQ_FIELD_INVOICE_EXPECTEDDATE, start) .end() @@ -657,7 +657,7 @@ WikittyQuery invoiceDebt = new WikittyQueryMaker().and() .parse(query) .containsOne(Invoice.FQ_FIELD_INVOICE_CUSTOMER) - .select(Element.ID).eq(Employee.FQ_FIELD_EMPLOYEE_COMPANY, companyId) + .select(Element.ID).or().ideq(companyId).eq(Employee.FQ_FIELD_EMPLOYEE_COMPANY, companyId).close() .close() .bw(Invoice.FQ_FIELD_INVOICE_EXPECTEDDATE, start, end) .end() @@ -667,7 +667,7 @@ WikittyQuery invoiceIncome = new WikittyQueryMaker().and() .parse(query) .containsOne(Invoice.FQ_FIELD_INVOICE_SUPPLIER) - .select(Element.ID).eq(Employee.FQ_FIELD_EMPLOYEE_COMPANY, companyId) + .select(Element.ID).or().ideq(companyId).eq(Employee.FQ_FIELD_EMPLOYEE_COMPANY, companyId).close() .close() .bw(Invoice.FQ_FIELD_INVOICE_EXPECTEDDATE, start, end) .end() @@ -787,7 +787,7 @@ Date d = getDate(invoice); String date = DateFormatUtils.format(d, budgetDateFormat); - Category c = invoice.getCategory(true); + Category c = invoice.getCategory(false); double amount = invoice.getAmount(); addAmount(date, c, amount);