This is an automated email from the git hooks/post-receive script. New commit to branch master in repository oipf-stub. See https://gitlab.nuiton.org/codelutin/oipf-stub.git commit 93c29453adc9fa3f765e11565a0e36b9e7980bfb Author: Samuel Maisonneuve <maisonneuve@codelutin.com> Date: Tue Apr 19 17:56:46 2016 +0200 [JsonTvProvider] - implementing a provider for json files (conversion of wml tv file into json) --- src/OipfDist.js | 4 +- src/provider/AbstractXmlTvProvider.js | 144 ++++++++++++++++++++++++ src/provider/JsonTvProvider.js | 199 ++++++++++++++++++++++++++++++++++ src/provider/XmlTvProvider.js | 121 +-------------------- 4 files changed, 352 insertions(+), 116 deletions(-) diff --git a/src/OipfDist.js b/src/OipfDist.js index c8b41e5..ce6203e 100644 --- a/src/OipfDist.js +++ b/src/OipfDist.js @@ -21,8 +21,10 @@ */ let OipfStubContext = require("../src/OipfStubContext"); let XmlTvProvider = require("../src/provider/XmlTvProvider.js"); +let JsonTvProvider = require("../src/provider/JsonTvProvider.js"); window.__oipf__ = { OipfStubContext, - XmlTvProvider + XmlTvProvider, + JsonTvProvider }; diff --git a/src/provider/AbstractXmlTvProvider.js b/src/provider/AbstractXmlTvProvider.js new file mode 100644 index 0000000..403a83b --- /dev/null +++ b/src/provider/AbstractXmlTvProvider.js @@ -0,0 +1,144 @@ +"use strict"; + +/* + * oipf-stub, (C) 2015 Code Lutin (SAS). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +let fs = require("fs"); + +let moment = require("moment"); +let DataProvider = require("./DataProvider.js"); +let JSZip = require("jszip"); +let JSZipUtils = require("jszip-utils"); +let defaultChannelInfo = require("../../data/fr_tnt_channels.json"); + +/* + * Provider interface fo all data provider. + */ +module.exports = class AbstractXmlTvProvider extends DataProvider { + + constructor(option) { + super(); + + option = option || {}; + this.maxDate = option.maxDate && moment(option.maxDate).unix(); // in second + this.endpoint = option.endpoint || "./data/fr_tnt.xml"; + this.channelInfo = {}; + if (option.acceptChannelWithoutDVB === false) { // undefined must be true + this.acceptChannelWithoutDVB = false; + } else { + this.acceptChannelWithoutDVB = true; + } + for (let info of option.channelInfo || defaultChannelInfo) { + this.channelInfo[info.name] = info; + } + + let storageKey = "oipf.provider.xmltv.cache"; + let oldData = localStorage.getItem(storageKey); + if (oldData) { + this.cache = JSON.parse(oldData); + } else { + this.cache = {data: {}}; + } + this.cache.storageKey = storageKey; + + let cacheExpiration = option.cacheExpiration || [1, "days"]; + if (Array.isArray(cacheExpiration)) { + this.cache.expiration = moment.duration.apply(moment.duration, cacheExpiration).asMilliseconds(); + } else { + this.cache.expiration = moment.duration(cacheExpiration).asMilliseconds(); + } + } + + /** + * Fetch local file content. + * + * @returns {Promise} fetch promise + */ + fetchLocalData() { + let endpoint = this.endpoint.replace(/^file:(\/\/)?/, ""); + let promise = new Promise(function(resolve, reject) { + fs.readFile(endpoint, function(err, data) { + if (err) { + reject(err); + } else if (endpoint.indexOf(".zip") !== -1) { + let zip = new JSZip(data), + files = zip.file(/^.*$/); + + resolve(files[0].asText()); + } else { + resolve(data.toString()); + } + }); + }); + return promise; + } + + /** + * Fetch an empty content. + * + * @returns {Promise} fetch promise + */ + fetchEmptyData() { + return new Promise(function(resolve) { resolve(""); }); + } + + /** + * Date must be formated as: YYYYMMDDHHmmss + */ + parseDateYYYYMMDDHHmmss(ds) { + let result = new Date( + ds.substring(0, 4), // Year + ds.substring(4, 6) - 1, // month + ds.substring(6, 8), // day + ds.substring(8, 10), // hour + ds.substring(10, 12), // minute + ds.substring(12, 14)); // second + return result; + } + + fetchData() { return Promise.reject(); } + + /** + * Parse channels from xml content. + * + * @returns {Promise} promise resolved with channels as json data + */ + getChannels() { + return this.fetchData() + .then(function(data) { + return data.channels; + }).catch(function(error) { + console.log("can't fetch", error); + return []; + }); + } + + /** + * Parse programme from xml content. + * + * @param {String} ccid channel ccid + * @returns {Promise} promise resolved with programmes as json data + */ + getProgrammes(index, channel) { + return this.fetchData() + .then(function(data) { + return data.programmes[channel.sourceID] || []; + }).catch(function(error) { + console.log("can't fetch", error); + return []; + }); + } + +}; diff --git a/src/provider/JsonTvProvider.js b/src/provider/JsonTvProvider.js new file mode 100644 index 0000000..f94a874 --- /dev/null +++ b/src/provider/JsonTvProvider.js @@ -0,0 +1,199 @@ +"use strict"; + +/* + * oipf-stub, (C) 2015 Code Lutin (SAS). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +let fs = require("fs"); + +let moment = require("moment"); +let AbstractXmlTvProvider = require("./AbstractXmlTvProvider.js"); +let JSZip = require("jszip"); +let JSZipUtils = require("jszip-utils"); +let defaultChannelInfo = require("../../data/fr_tnt_channels.json"); + +/* + * Provider interface fo all data provider. + */ +module.exports = class JsonTvProvider extends AbstractXmlTvProvider { + + /** + * Fetch json content depending on endpoint configuration (local file or remote url). + * + * @returns {Promise} fetch promise + */ + fetchJsonData() { + let result; + if (this.endpoint === "" || this.endpoint.indexOf("none") === 0) { + result = this.fetchEmptyData(); + } else if (this.endpoint.indexOf("file:") === 0) { + result = this.fetchLocalData().then(function(text) { + this.processData(text); + }); + } else { + result = this.fetchRemoteData(); + } + return result; + } + + /** + * Fetch remote endpoint url. + * + * @returns {Promise} fetch promise + */ + fetchRemoteData() { + let promise; + + let endpoint = this.endpoint; + if (endpoint.indexOf(".zip") !== -1) { + promise = new Promise(function(resolve, reject) { + JSZipUtils.getBinaryContent(endpoint, function(err, data) { + if (err) { + reject(err); + } else { + try { + let zip = new JSZip(data); + let files = zip.file(/^.*\.json$/); + let json = files[0].asText(); // use first xml file + resolve(json); + } catch (e) { + reject(e); + } + } + }); + }).then(function(json) { + return this.processData(json); + }); + } else { + promise = fetch(endpoint) + .then(function(response) { + return response.json(); + }).then((json) => this.processData(json)); + } + return promise; + } + + processData(json) { + // no data or need refresh for next call + let maxDate = this.maxDate; + let channelInfo = this.channelInfo; + let acceptChannelWithoutDVB = this.acceptChannelWithoutDVB; + let parseDate = this.parseDateYYYYMMDDHHmmss; + + console.log("################### LOAD JSON DATA"); + let time = Date.now(); + + let channels = []; + let programmes = {}; + let done = {count: 0}; // data source has same programme many time, result must return only one + + let jsonChannels = json.channel, jsonChannel; + for (let i = 0, l = jsonChannels.length; i < l; i++) { + jsonChannel = jsonChannels[i]; + + // Process channels + const channel = {sourceID: jsonChannel["@attributes"].id, idType: 30}; + + // can be "1", "C1.telerama.fr" + const sourceID = channel.sourceID.replace(/\D/g, ""); + channel.majorChannel = parseInt(sourceID, 10); + channel.name = jsonChannel["display-name"]; + + const info = channelInfo[channel.name + " HD"] || channelInfo[channel.name]; + if (acceptChannelWithoutDVB || info) { + Object.assign(channel, info || {}); + channels.push(channel); + channels["cid-" + channel.sourceID] = channel; + } + } + + // Process programmes + let jsonProgrammes = json.programme, jsonProgramme, jsonProgrammeAttrs; + for (let i = 0, l = jsonProgrammes.length; i < l; i++) { + jsonProgramme = jsonProgrammes[i]; + jsonProgrammeAttrs = jsonProgramme["@attributes"]; + let startDate = parseDate(jsonProgrammeAttrs.start).valueOf() / 1000; // in second + // on some xml tv files, programmes are not ordered by dates + if (startDate <= maxDate) { + console.log("programme found"); + let key = jsonProgrammeAttrs.channel + "-" + jsonProgrammeAttrs.start; + if (!done[key]) { + done[key] = true; + done.count++; + let channelId = jsonProgrammeAttrs.channel; + let prog = {}; + let progs = programmes[channelId] = programmes[channelId] || []; + progs.push(prog); + + // channel + prog.channel = channels["cid-" + channelId]; + + // parse dates + let endDate = parseDate(jsonProgrammeAttrs.stop).valueOf() / 1000; // in second + prog.startTime = startDate; + prog.duration = endDate - startDate; + prog.name = jsonProgramme.title; + prog.subtitle = jsonProgramme["sub-title"]; + prog.description = jsonProgramme.desc; + if (jsonProgramme.category) { + for (let j = 0, m = jsonProgramme.category.length; j < m; j++) { + prog.genre = prog.genre || []; + prog.genre.push(jsonProgramme.category[j]); + } + } + prog._logoURL = jsonProgramme.icon && jsonProgramme.icon["@attributes"].src; + } + } + } + + console.log("################### time: ", Date.now() - time, "progs: ", done.count); + + // sort because json file order matters + channels = channels.sort(function(c1, c2) { + return c1.minorChannel - c2.minorChannel; + }); + + // Storing result + const cache = this.cache; + cache.data[this.endpoint] = {date: Date.now(), data: {channels, programmes}}; + localStorage.setItem(cache.storageKey, JSON.stringify(cache)); + + return {channels, programmes}; + } + + fetchData() { + let result; + let cache = this.cache; + let cacheKey = this.endpoint; + let value = cache.data[cacheKey] || {}; + + if (value.promise && !value.data) { + result = value.promise; + } else { + if (value.data) { + result = Promise.resolve(value.data); + } + + if (!value.data || value.date + cache.expiration < Date.now()) { + let promise = this.fetchJsonData(); + + result = result || promise; + cache.data[cacheKey] = {promise, date: value.date, data: value.data}; + } + } + + return result; + } + +}; diff --git a/src/provider/XmlTvProvider.js b/src/provider/XmlTvProvider.js index 0f5dbd3..c3cc59a 100644 --- a/src/provider/XmlTvProvider.js +++ b/src/provider/XmlTvProvider.js @@ -18,7 +18,7 @@ let fs = require("fs"); let moment = require("moment"); -let DataProvider = require("./DataProvider.js"); +let AbstractXmlTvProvider = require("./AbstractXmlTvProvider.js"); let JSZip = require("jszip"); let JSZipUtils = require("jszip-utils"); let XMLParser = require("exml-easysax"); @@ -27,40 +27,7 @@ let defaultChannelInfo = require("../../data/fr_tnt_channels.json"); /* * Provider interface fo all data provider. */ -module.exports = class XmlTvProvider extends DataProvider { - - constructor(option) { - super(); - - option = option || {}; - this.maxDate = option.maxDate && moment(option.maxDate).unix(); // in second - this.endpoint = option.endpoint || "./data/fr_tnt.xml"; - this.channelInfo = {}; - if (option.acceptChannelWithoutDVB === false) { // undefined must be true - this.acceptChannelWithoutDVB = false; - } else { - this.acceptChannelWithoutDVB = true; - } - for (let info of option.channelInfo || defaultChannelInfo) { - this.channelInfo[info.name] = info; - } - - let storageKey = "oipf.provider.xmltv.cache"; - let oldData = localStorage.getItem(storageKey); - if (oldData) { - this.cache = JSON.parse(oldData); - } else { - this.cache = {data: {}}; - } - this.cache.storageKey = storageKey; - - let cacheExpiration = option.cacheExpiration || [1, "days"]; - if (Array.isArray(cacheExpiration)) { - this.cache.expiration = moment.duration.apply(moment.duration, cacheExpiration).asMilliseconds(); - } else { - this.cache.expiration = moment.duration(cacheExpiration).asMilliseconds(); - } - } +module.exports = class XmlTvProvider extends AbstractXmlTvProvider { /** * Fetch xml content depending on endpoint configuration (local file or remote url). @@ -129,7 +96,8 @@ module.exports = class XmlTvProvider extends DataProvider { }); }; - fetch(endpoint).then(function(response) { + fetch(endpoint) + .then(function(response) { read(response.body.getReader()); }).catch(function(e) { reject(e); @@ -139,53 +107,6 @@ module.exports = class XmlTvProvider extends DataProvider { return promise; } - /** - * Fetch local file content. - * - * @returns {Promise} fetch promise - */ - fetchLocalData() { - let endpoint = this.endpoint.replace(/^file:(\/\/)?/, ""); - let promise = new Promise(function(resolve, reject) { - fs.readFile(endpoint, function(err, data) { - if (err) { - reject(err); - } else if (endpoint.indexOf(".zip") !== -1) { - let zip = new JSZip(data), - files = zip.file(/^.*$/); - - resolve(files[0].asText()); - } else { - resolve(data.toString()); - } - }); - }); - return promise; - } - - /** - * Fetch an empty content. - * - * @returns {Promise} fetch promise - */ - fetchEmptyData() { - return new Promise(function(resolve) { resolve(""); }); - } - - /** - * Date must be formated as: YYYYMMDDHHmmss - */ - parseDateYYYYMMDDHHmmss(ds) { - let result = new Date( - ds.substring(0, 4), // Year - ds.substring(4, 6) - 1, // month - ds.substring(6, 8), // day - ds.substring(8, 10), // hour - ds.substring(10, 12), // minute - ds.substring(12, 14)); // second - return result; - } - fetchData() { let result; let cache = this.cache; @@ -234,9 +155,11 @@ module.exports = class XmlTvProvider extends DataProvider { }); parser.on("tv/programme", function(attrs) { + let startDate = parseDate(attrs.start).valueOf() / 1000; // in second // on some xml tv files, programmes are not ordered by dates if (startDate <= maxDate) { + console.log("programme found"); let key = attrs.channel + "-" + attrs.start; if (!done[key]) { done[key] = true; @@ -300,36 +223,4 @@ module.exports = class XmlTvProvider extends DataProvider { return result; } - - /** - * Parse channels from xml content. - * - * @returns {Promise} promise resolved with channels as json data - */ - getChannels() { - return this.fetchData() - .then(function(data) { - return data.channels; - }).catch(function(error) { - console.log("can't fetch", error); - return []; - }); - } - - /** - * Parse programme from xml content. - * - * @param {String} ccid channel ccid - * @returns {Promise} promise resolved with programmes as json data - */ - getProgrammes(index, channel) { - return this.fetchData() - .then(function(data) { - return data.programmes[channel.sourceID] || []; - }).catch(function(error) { - console.log("can't fetch", error); - return []; - }); - } - }; -- To stop receiving notification emails like this one, please contact SCM administrator <admin+scm@forge.codelutin.com>.