branch master updated (f874c86 -> 93c2945)
This is an automated email from the git hooks/post-receive script. New change to branch master in repository oipf-stub. See https://gitlab.nuiton.org/codelutin/oipf-stub.git from f874c86 Fix event priority sort during key strategy new 93c2945 [JsonTvProvider] - implementing a provider for json files (conversion of wml tv file into json) The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit 93c29453adc9fa3f765e11565a0e36b9e7980bfb Author: Samuel Maisonneuve <maisonneuve@codelutin.com> Date: Tue Apr 19 17:56:46 2016 +0200 [JsonTvProvider] - implementing a provider for json files (conversion of wml tv file into json) Summary of changes: src/OipfDist.js | 4 +- src/provider/AbstractXmlTvProvider.js | 144 ++++++++++++++++++++++++ src/provider/JsonTvProvider.js | 199 ++++++++++++++++++++++++++++++++++ src/provider/XmlTvProvider.js | 121 +-------------------- 4 files changed, 352 insertions(+), 116 deletions(-) create mode 100644 src/provider/AbstractXmlTvProvider.js create mode 100644 src/provider/JsonTvProvider.js -- To stop receiving notification emails like this one, please contact SCM administrator <admin+scm@forge.codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch master in repository oipf-stub. See https://gitlab.nuiton.org/codelutin/oipf-stub.git commit 93c29453adc9fa3f765e11565a0e36b9e7980bfb Author: Samuel Maisonneuve <maisonneuve@codelutin.com> Date: Tue Apr 19 17:56:46 2016 +0200 [JsonTvProvider] - implementing a provider for json files (conversion of wml tv file into json) --- src/OipfDist.js | 4 +- src/provider/AbstractXmlTvProvider.js | 144 ++++++++++++++++++++++++ src/provider/JsonTvProvider.js | 199 ++++++++++++++++++++++++++++++++++ src/provider/XmlTvProvider.js | 121 +-------------------- 4 files changed, 352 insertions(+), 116 deletions(-) diff --git a/src/OipfDist.js b/src/OipfDist.js index c8b41e5..ce6203e 100644 --- a/src/OipfDist.js +++ b/src/OipfDist.js @@ -21,8 +21,10 @@ */ let OipfStubContext = require("../src/OipfStubContext"); let XmlTvProvider = require("../src/provider/XmlTvProvider.js"); +let JsonTvProvider = require("../src/provider/JsonTvProvider.js"); window.__oipf__ = { OipfStubContext, - XmlTvProvider + XmlTvProvider, + JsonTvProvider }; diff --git a/src/provider/AbstractXmlTvProvider.js b/src/provider/AbstractXmlTvProvider.js new file mode 100644 index 0000000..403a83b --- /dev/null +++ b/src/provider/AbstractXmlTvProvider.js @@ -0,0 +1,144 @@ +"use strict"; + +/* + * oipf-stub, (C) 2015 Code Lutin (SAS). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +let fs = require("fs"); + +let moment = require("moment"); +let DataProvider = require("./DataProvider.js"); +let JSZip = require("jszip"); +let JSZipUtils = require("jszip-utils"); +let defaultChannelInfo = require("../../data/fr_tnt_channels.json"); + +/* + * Provider interface fo all data provider. + */ +module.exports = class AbstractXmlTvProvider extends DataProvider { + + constructor(option) { + super(); + + option = option || {}; + this.maxDate = option.maxDate && moment(option.maxDate).unix(); // in second + this.endpoint = option.endpoint || "./data/fr_tnt.xml"; + this.channelInfo = {}; + if (option.acceptChannelWithoutDVB === false) { // undefined must be true + this.acceptChannelWithoutDVB = false; + } else { + this.acceptChannelWithoutDVB = true; + } + for (let info of option.channelInfo || defaultChannelInfo) { + this.channelInfo[info.name] = info; + } + + let storageKey = "oipf.provider.xmltv.cache"; + let oldData = localStorage.getItem(storageKey); + if (oldData) { + this.cache = JSON.parse(oldData); + } else { + this.cache = {data: {}}; + } + this.cache.storageKey = storageKey; + + let cacheExpiration = option.cacheExpiration || [1, "days"]; + if (Array.isArray(cacheExpiration)) { + this.cache.expiration = moment.duration.apply(moment.duration, cacheExpiration).asMilliseconds(); + } else { + this.cache.expiration = moment.duration(cacheExpiration).asMilliseconds(); + } + } + + /** + * Fetch local file content. + * + * @returns {Promise} fetch promise + */ + fetchLocalData() { + let endpoint = this.endpoint.replace(/^file:(\/\/)?/, ""); + let promise = new Promise(function(resolve, reject) { + fs.readFile(endpoint, function(err, data) { + if (err) { + reject(err); + } else if (endpoint.indexOf(".zip") !== -1) { + let zip = new JSZip(data), + files = zip.file(/^.*$/); + + resolve(files[0].asText()); + } else { + resolve(data.toString()); + } + }); + }); + return promise; + } + + /** + * Fetch an empty content. + * + * @returns {Promise} fetch promise + */ + fetchEmptyData() { + return new Promise(function(resolve) { resolve(""); }); + } + + /** + * Date must be formated as: YYYYMMDDHHmmss + */ + parseDateYYYYMMDDHHmmss(ds) { + let result = new Date( + ds.substring(0, 4), // Year + ds.substring(4, 6) - 1, // month + ds.substring(6, 8), // day + ds.substring(8, 10), // hour + ds.substring(10, 12), // minute + ds.substring(12, 14)); // second + return result; + } + + fetchData() { return Promise.reject(); } + + /** + * Parse channels from xml content. + * + * @returns {Promise} promise resolved with channels as json data + */ + getChannels() { + return this.fetchData() + .then(function(data) { + return data.channels; + }).catch(function(error) { + console.log("can't fetch", error); + return []; + }); + } + + /** + * Parse programme from xml content. + * + * @param {String} ccid channel ccid + * @returns {Promise} promise resolved with programmes as json data + */ + getProgrammes(index, channel) { + return this.fetchData() + .then(function(data) { + return data.programmes[channel.sourceID] || []; + }).catch(function(error) { + console.log("can't fetch", error); + return []; + }); + } + +}; diff --git a/src/provider/JsonTvProvider.js b/src/provider/JsonTvProvider.js new file mode 100644 index 0000000..f94a874 --- /dev/null +++ b/src/provider/JsonTvProvider.js @@ -0,0 +1,199 @@ +"use strict"; + +/* + * oipf-stub, (C) 2015 Code Lutin (SAS). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +let fs = require("fs"); + +let moment = require("moment"); +let AbstractXmlTvProvider = require("./AbstractXmlTvProvider.js"); +let JSZip = require("jszip"); +let JSZipUtils = require("jszip-utils"); +let defaultChannelInfo = require("../../data/fr_tnt_channels.json"); + +/* + * Provider interface fo all data provider. + */ +module.exports = class JsonTvProvider extends AbstractXmlTvProvider { + + /** + * Fetch json content depending on endpoint configuration (local file or remote url). + * + * @returns {Promise} fetch promise + */ + fetchJsonData() { + let result; + if (this.endpoint === "" || this.endpoint.indexOf("none") === 0) { + result = this.fetchEmptyData(); + } else if (this.endpoint.indexOf("file:") === 0) { + result = this.fetchLocalData().then(function(text) { + this.processData(text); + }); + } else { + result = this.fetchRemoteData(); + } + return result; + } + + /** + * Fetch remote endpoint url. + * + * @returns {Promise} fetch promise + */ + fetchRemoteData() { + let promise; + + let endpoint = this.endpoint; + if (endpoint.indexOf(".zip") !== -1) { + promise = new Promise(function(resolve, reject) { + JSZipUtils.getBinaryContent(endpoint, function(err, data) { + if (err) { + reject(err); + } else { + try { + let zip = new JSZip(data); + let files = zip.file(/^.*\.json$/); + let json = files[0].asText(); // use first xml file + resolve(json); + } catch (e) { + reject(e); + } + } + }); + }).then(function(json) { + return this.processData(json); + }); + } else { + promise = fetch(endpoint) + .then(function(response) { + return response.json(); + }).then((json) => this.processData(json)); + } + return promise; + } + + processData(json) { + // no data or need refresh for next call + let maxDate = this.maxDate; + let channelInfo = this.channelInfo; + let acceptChannelWithoutDVB = this.acceptChannelWithoutDVB; + let parseDate = this.parseDateYYYYMMDDHHmmss; + + console.log("################### LOAD JSON DATA"); + let time = Date.now(); + + let channels = []; + let programmes = {}; + let done = {count: 0}; // data source has same programme many time, result must return only one + + let jsonChannels = json.channel, jsonChannel; + for (let i = 0, l = jsonChannels.length; i < l; i++) { + jsonChannel = jsonChannels[i]; + + // Process channels + const channel = {sourceID: jsonChannel["@attributes"].id, idType: 30}; + + // can be "1", "C1.telerama.fr" + const sourceID = channel.sourceID.replace(/\D/g, ""); + channel.majorChannel = parseInt(sourceID, 10); + channel.name = jsonChannel["display-name"]; + + const info = channelInfo[channel.name + " HD"] || channelInfo[channel.name]; + if (acceptChannelWithoutDVB || info) { + Object.assign(channel, info || {}); + channels.push(channel); + channels["cid-" + channel.sourceID] = channel; + } + } + + // Process programmes + let jsonProgrammes = json.programme, jsonProgramme, jsonProgrammeAttrs; + for (let i = 0, l = jsonProgrammes.length; i < l; i++) { + jsonProgramme = jsonProgrammes[i]; + jsonProgrammeAttrs = jsonProgramme["@attributes"]; + let startDate = parseDate(jsonProgrammeAttrs.start).valueOf() / 1000; // in second + // on some xml tv files, programmes are not ordered by dates + if (startDate <= maxDate) { + console.log("programme found"); + let key = jsonProgrammeAttrs.channel + "-" + jsonProgrammeAttrs.start; + if (!done[key]) { + done[key] = true; + done.count++; + let channelId = jsonProgrammeAttrs.channel; + let prog = {}; + let progs = programmes[channelId] = programmes[channelId] || []; + progs.push(prog); + + // channel + prog.channel = channels["cid-" + channelId]; + + // parse dates + let endDate = parseDate(jsonProgrammeAttrs.stop).valueOf() / 1000; // in second + prog.startTime = startDate; + prog.duration = endDate - startDate; + prog.name = jsonProgramme.title; + prog.subtitle = jsonProgramme["sub-title"]; + prog.description = jsonProgramme.desc; + if (jsonProgramme.category) { + for (let j = 0, m = jsonProgramme.category.length; j < m; j++) { + prog.genre = prog.genre || []; + prog.genre.push(jsonProgramme.category[j]); + } + } + prog._logoURL = jsonProgramme.icon && jsonProgramme.icon["@attributes"].src; + } + } + } + + console.log("################### time: ", Date.now() - time, "progs: ", done.count); + + // sort because json file order matters + channels = channels.sort(function(c1, c2) { + return c1.minorChannel - c2.minorChannel; + }); + + // Storing result + const cache = this.cache; + cache.data[this.endpoint] = {date: Date.now(), data: {channels, programmes}}; + localStorage.setItem(cache.storageKey, JSON.stringify(cache)); + + return {channels, programmes}; + } + + fetchData() { + let result; + let cache = this.cache; + let cacheKey = this.endpoint; + let value = cache.data[cacheKey] || {}; + + if (value.promise && !value.data) { + result = value.promise; + } else { + if (value.data) { + result = Promise.resolve(value.data); + } + + if (!value.data || value.date + cache.expiration < Date.now()) { + let promise = this.fetchJsonData(); + + result = result || promise; + cache.data[cacheKey] = {promise, date: value.date, data: value.data}; + } + } + + return result; + } + +}; diff --git a/src/provider/XmlTvProvider.js b/src/provider/XmlTvProvider.js index 0f5dbd3..c3cc59a 100644 --- a/src/provider/XmlTvProvider.js +++ b/src/provider/XmlTvProvider.js @@ -18,7 +18,7 @@ let fs = require("fs"); let moment = require("moment"); -let DataProvider = require("./DataProvider.js"); +let AbstractXmlTvProvider = require("./AbstractXmlTvProvider.js"); let JSZip = require("jszip"); let JSZipUtils = require("jszip-utils"); let XMLParser = require("exml-easysax"); @@ -27,40 +27,7 @@ let defaultChannelInfo = require("../../data/fr_tnt_channels.json"); /* * Provider interface fo all data provider. */ -module.exports = class XmlTvProvider extends DataProvider { - - constructor(option) { - super(); - - option = option || {}; - this.maxDate = option.maxDate && moment(option.maxDate).unix(); // in second - this.endpoint = option.endpoint || "./data/fr_tnt.xml"; - this.channelInfo = {}; - if (option.acceptChannelWithoutDVB === false) { // undefined must be true - this.acceptChannelWithoutDVB = false; - } else { - this.acceptChannelWithoutDVB = true; - } - for (let info of option.channelInfo || defaultChannelInfo) { - this.channelInfo[info.name] = info; - } - - let storageKey = "oipf.provider.xmltv.cache"; - let oldData = localStorage.getItem(storageKey); - if (oldData) { - this.cache = JSON.parse(oldData); - } else { - this.cache = {data: {}}; - } - this.cache.storageKey = storageKey; - - let cacheExpiration = option.cacheExpiration || [1, "days"]; - if (Array.isArray(cacheExpiration)) { - this.cache.expiration = moment.duration.apply(moment.duration, cacheExpiration).asMilliseconds(); - } else { - this.cache.expiration = moment.duration(cacheExpiration).asMilliseconds(); - } - } +module.exports = class XmlTvProvider extends AbstractXmlTvProvider { /** * Fetch xml content depending on endpoint configuration (local file or remote url). @@ -129,7 +96,8 @@ module.exports = class XmlTvProvider extends DataProvider { }); }; - fetch(endpoint).then(function(response) { + fetch(endpoint) + .then(function(response) { read(response.body.getReader()); }).catch(function(e) { reject(e); @@ -139,53 +107,6 @@ module.exports = class XmlTvProvider extends DataProvider { return promise; } - /** - * Fetch local file content. - * - * @returns {Promise} fetch promise - */ - fetchLocalData() { - let endpoint = this.endpoint.replace(/^file:(\/\/)?/, ""); - let promise = new Promise(function(resolve, reject) { - fs.readFile(endpoint, function(err, data) { - if (err) { - reject(err); - } else if (endpoint.indexOf(".zip") !== -1) { - let zip = new JSZip(data), - files = zip.file(/^.*$/); - - resolve(files[0].asText()); - } else { - resolve(data.toString()); - } - }); - }); - return promise; - } - - /** - * Fetch an empty content. - * - * @returns {Promise} fetch promise - */ - fetchEmptyData() { - return new Promise(function(resolve) { resolve(""); }); - } - - /** - * Date must be formated as: YYYYMMDDHHmmss - */ - parseDateYYYYMMDDHHmmss(ds) { - let result = new Date( - ds.substring(0, 4), // Year - ds.substring(4, 6) - 1, // month - ds.substring(6, 8), // day - ds.substring(8, 10), // hour - ds.substring(10, 12), // minute - ds.substring(12, 14)); // second - return result; - } - fetchData() { let result; let cache = this.cache; @@ -234,9 +155,11 @@ module.exports = class XmlTvProvider extends DataProvider { }); parser.on("tv/programme", function(attrs) { + let startDate = parseDate(attrs.start).valueOf() / 1000; // in second // on some xml tv files, programmes are not ordered by dates if (startDate <= maxDate) { + console.log("programme found"); let key = attrs.channel + "-" + attrs.start; if (!done[key]) { done[key] = true; @@ -300,36 +223,4 @@ module.exports = class XmlTvProvider extends DataProvider { return result; } - - /** - * Parse channels from xml content. - * - * @returns {Promise} promise resolved with channels as json data - */ - getChannels() { - return this.fetchData() - .then(function(data) { - return data.channels; - }).catch(function(error) { - console.log("can't fetch", error); - return []; - }); - } - - /** - * Parse programme from xml content. - * - * @param {String} ccid channel ccid - * @returns {Promise} promise resolved with programmes as json data - */ - getProgrammes(index, channel) { - return this.fetchData() - .then(function(data) { - return data.programmes[channel.sourceID] || []; - }).catch(function(error) { - console.log("can't fetch", error); - return []; - }); - } - }; -- To stop receiving notification emails like this one, please contact SCM administrator <admin+scm@forge.codelutin.com>.
participants (1)
-
scm