diff --git a/src/controller.js b/src/controller.js index 20ddee0..a0567ec 100644 --- a/src/controller.js +++ b/src/controller.js @@ -1,4 +1,4 @@ -import { FrontMatter } from './front-matter'; +import { FrontMatter, mergeFromYMLFrontmatter } from './front-matter'; import { DMath } from './components/d-math'; import { collect_citations } from './helpers/citation.js'; import { parseFrontmatter } from './components/d-front-matter'; @@ -107,7 +107,7 @@ export const Controller = { onFrontMatterChanged(event) { const data = event.detail; - frontMatter.mergeFromYMLFrontmatter(data); + mergeFromYMLFrontmatter(frontMatter, data); const interstitial = document.querySelector('d-interstitial'); if (interstitial) { diff --git a/src/extractors/citations.js b/src/extractors/citations.js index ce2a504..4d9e8c0 100644 --- a/src/extractors/citations.js +++ b/src/extractors/citations.js @@ -1,5 +1,10 @@ import { collect_citations } from '../helpers/citation.js'; export default function(dom, data) { - data.citations = collect_citations(dom); + const citations = new Set(data.citations); + const newCitations = collect_citations(dom); + for (const citation of newCitations) { + citations.add(citation); + } + data.citations = Array.from(citations); } diff --git a/src/extractors/front-matter.js b/src/extractors/front-matter.js index 1cbdbfb..1892a62 100644 --- a/src/extractors/front-matter.js +++ b/src/extractors/front-matter.js @@ -1,4 +1,5 @@ import { parseFrontmatter } from '../components/d-front-matter'; +import { mergeFromYMLFrontmatter } from '../front-matter.js'; export default function(dom, data) { const frontMatterTag = dom.querySelector('d-front-matter'); @@ -7,5 +8,5 @@ export default function(dom, data) { return; } const extractedData = parseFrontmatter(frontMatterTag); - data.mergeFromYMLFrontmatter(extractedData); + mergeFromYMLFrontmatter(data, extractedData); } diff --git a/src/front-matter.js b/src/front-matter.js index eed187e..81766fa 100644 --- a/src/front-matter.js +++ b/src/front-matter.js @@ -42,6 +42,15 @@ class Author { } } +export function mergeFromYMLFrontmatter(target, source) { + target.title = source.title; + target.publishedDate = new Date(source.published); + target.description = source.description; + target.authors = source.authors.map( (authorObject) => new Author(authorObject)); + target.katex = source.katex; + target.password = source.password; +} + export class FrontMatter { constructor() { this.title = ''; // 'Attention and Augmented Recurrent Neural Networks' @@ -118,15 +127,6 @@ export class FrontMatter { // - Google Brain: // - Google Brain: http://g.co/brain - mergeFromYMLFrontmatter(data) { - this.title = data.title; - this.publishedDate = new Date(data.published); - this.description = data.description; - this.authors = data.authors.map( (authorObject) => new Author(authorObject)); - this.katex = data.katex; - this.password = data.password; - } - // // Computed Properties // diff --git a/src/transforms.js b/src/transforms.js index c3b778c..f03a7c4 100644 --- a/src/transforms.js +++ b/src/transforms.js @@ -7,11 +7,11 @@ import ExtractFrontmatter from './extractors/front-matter'; import ExtractBibliography from './extractors/bibliography'; import ExtractCitations from './extractors/citations'; -const extractors = [ - ExtractFrontmatter, - ExtractBibliography, - ExtractCitations, -]; +const extractors = new Map([ + ['ExtractFrontmatter', ExtractFrontmatter], + ['ExtractBibliography', ExtractBibliography], + ['ExtractCitations', ExtractCitations], +]); /* Transforms */ import HTML from './transforms/html'; @@ -25,43 +25,53 @@ import TOC from './transforms/toc'; import Typeset from './transforms/typeset'; import Bibliography from './transforms/bibliography'; -const transforms = [ - HTML, makeStyleTag, Polyfills, OptionalComponents, TOC, Byline, Mathematics, - Meta, Typeset, Bibliography, -]; +const transforms = new Map([ + ['HTML', HTML], + ['makeStyleTag', makeStyleTag], + ['Polyfills', Polyfills], + ['OptionalComponents', OptionalComponents], + ['TOC', TOC], + ['Byline', Byline], + ['Mathematics', Mathematics], + ['Meta', Meta], + ['Typeset', Typeset], + ['Bibliography', Bibliography], +]); /* Distill Transforms */ import DistillHeader from './distill-transforms/distill-header'; import DistillAppendix from './distill-transforms/distill-appendix'; import DistillFooter from './distill-transforms/distill-footer'; -const distillTransforms = [ - DistillHeader, DistillAppendix, DistillFooter, -]; +const distillTransforms = new Map([ + ['DistillHeader', DistillHeader], + ['DistillAppendix', DistillAppendix], + ['DistillFooter', DistillFooter], +]); /* Exported functions */ -export function render(dom, data) { +export function render(dom, data, verbose=true) { // first, we collect static data from the dom - for (const extract of extractors) { - console.warn('Running extractor...'); - extract(dom, data); + for (const [name, extract] of extractors.entries()) { + if (verbose) console.warn('Running extractor: ' + name); + extract(dom, data, verbose); } // secondly we use it to transform parts of the dom - for (const transform of transforms) { - console.warn('Running transform...'); + for (const [name, transform] of transforms.entries()) { + if (verbose) console.warn('Running transform: ' + name); // console.warn('Running transform: ', transform); - transform(dom, data); + transform(dom, data, verbose); } dom.body.setAttribute('distill-prerendered', ''); // the function calling us can now use the transformed dom and filled data object } -export function distillify(dom, data) { +export function distillify(dom, data, verbose=true) { // thirdly, we can use these additional transforms when publishing on the Distill website - for (const transform of distillTransforms) { - // console.warn('Running distillify: ', transform); - transform(dom, data); + for (const [name, transform] of distillTransforms.entries()) { + if (verbose) console.warn('Running distillify: ', name); + transform(dom, data, verbose); } } @@ -70,14 +80,12 @@ export function usesTemplateV2(dom) { let usesV2 = undefined; for (const tag of tags) { const src = tag.src; - if (src.includes('distill.pub')) { - if (src.includes('template.v1.js')) { - usesV2 = false; - } else if (src.includes('template.v2.js')) { - usesV2 = true; - } else { - throw new Error('Uses distill template, but unknown version?!'); - } + if (src.includes('template.v1.js')) { + usesV2 = false; + } else if (src.includes('template.v2.js')) { + usesV2 = true; + } else if (src.includes('template.')) { + throw new Error('Uses distill template, but unknown version?!'); } } @@ -87,5 +95,10 @@ export function usesTemplateV2(dom) { return usesV2; } } +export { FrontMatter }; // TODO: removable? -export { FrontMatter }; +export const testing = { + extractors: extractors, + transforms: transforms, + distillTransforms: distillTransforms +}; diff --git a/src/transforms/mathematics.js b/src/transforms/mathematics.js index a498b1a..88d6052 100644 --- a/src/transforms/mathematics.js +++ b/src/transforms/mathematics.js @@ -5,6 +5,11 @@ export default function(dom, data) { let needsCSS = false; const article = dom.querySelector('d-article'); + if (!article) { + console.warn("No d-article tag found!"); + return; + } + if (data.katex && data.katex.delimiters) { global.document = dom; renderMathInElement(article, data.katex); diff --git a/src/transforms/meta.js b/src/transforms/meta.js index 8119439..5f878fd 100644 --- a/src/transforms/meta.js +++ b/src/transforms/meta.js @@ -22,7 +22,8 @@ export default function(dom, data) { `); - data.authors.forEach((a) => { + + (data.authors || []).forEach((a) => { appendHtml(head, ` `); }); @@ -58,7 +59,7 @@ export default function(dom, data) { meta('citation_fulltext_html_url', data.url); meta('citation_volume', data.volume); meta('citation_issue', data.issue); - meta('citation_firstpage', data.doiSuffix? `e${data.doiSuffix}` : undefined); + meta('citation_firstpage', data.doiSuffix ? `e${data.doiSuffix}` : undefined); meta('citation_doi', data.doi); let journal = data.journal || {}; @@ -75,17 +76,21 @@ export default function(dom, data) { meta('citation_author', `${a.lastName}, ${a.firstName}`); meta('citation_author_institution', a.affiliation); }); + } else { + console.warn('No DOI suffix in data; not adding citation meta tags!'); + } - if (data.citations) { - data.citations.forEach(key => { - let d = data.bibliography[key]; - if(!d) { - console.warn('No bibliography data fround for ' + key); - } else { - meta('citation_reference', citation_meta_content(data.bibliography[key]) ); - } - }); - } + if (data.citations) { + data.citations.forEach(key => { + if (data.bibliography && data.bibliography.has(key)) { + const entry = data.bibliography.get(key); + meta('citation_reference', citation_meta_content(entry) ); + } else { + console.warn('No bibliography data found for ' + key); + } + }); + } else { + console.warn('No citations found; not adding any references meta tags!'); } } diff --git a/src/transforms/optional-components.js b/src/transforms/optional-components.js index bbe30de..410dfbc 100644 --- a/src/transforms/optional-components.js +++ b/src/transforms/optional-components.js @@ -9,7 +9,11 @@ export default function(dom, data) { const article = dom.querySelector('d-article'); - // const abstract = dom.querySelector('d-abstract'); + + if (!article) { + console.warn('No d-article tag found!'); + return; + } let interstitial = dom.querySelector('d-interstitial'); if (!interstitial && data.password) { diff --git a/test/helpers.js b/test/helpers.js new file mode 100644 index 0000000..5e6fec1 --- /dev/null +++ b/test/helpers.js @@ -0,0 +1,3 @@ +function nodeListToArray(nodeList) { + return Array.prototype.slice.call(nodeList); +} diff --git a/test/transforms.js b/test/transforms.js index 3fd6143..2808549 100644 --- a/test/transforms.js +++ b/test/transforms.js @@ -1,15 +1,23 @@ -/* global it, should, describe, before, beforeEach, after, afterEach */ +/* global it, describe, before, beforeEach, after, afterEach */ -const jsdom = require("jsdom"); +const jsdom = require('jsdom'); const { JSDOM } = jsdom; const expect = require('chai').expect; const distill = require('../dist/transforms.v2.js'); +// omitJSDOMErrors as JSDOM routinely can't parse modern CSS +const virtualConsole = new jsdom.VirtualConsole(); +virtualConsole.sendTo(console, { omitJSDOMErrors: true }); +const options = { runScripts: 'outside-only', QuerySelector: true, virtualConsole: virtualConsole }; + describe('Distill V2 (transforms)', function() { - it('should export usesTemplateV2()', function() { + it('should export its expected interface', function() { + expect(distill.testing).to.be.an('object'); expect(distill.usesTemplateV2).to.be.a('function'); + expect(distill.render).to.be.a('function'); + expect(distill.distillify).to.be.a('function'); }); describe('#usesTemplateV2()', function() { @@ -24,8 +32,13 @@ describe('Distill V2 (transforms)', function() { expect(distill.usesTemplateV2(frag)).to.be.true; }); + it('should detect local scripts as well', function() { + const frag = JSDOM.fragment(''); + expect(distill.usesTemplateV2(frag)).to.be.true; + }); + it('should error on unknown distill script', function() { - const frag = JSDOM.fragment(''); + const frag = JSDOM.fragment(''); expect(()=> distill.usesTemplateV2(frag)).to.throw('unknown'); }); @@ -36,18 +49,146 @@ describe('Distill V2 (transforms)', function() { }); - - it('should export render()', function() { - expect(distill.render).to.be.a('function'); - }); - describe('#render()', function() { - it('should extract metadata'); - it('should run transforms'); + describe('should extract metadata', function() { - }); + it('should extract citations', function() { + const dom = new JSDOM('sth', options); + const data = {}; + const extractCitations = distill.testing.extractors.get('ExtractCitations'); + expect(extractCitations).to.be.a('function'); + extractCitations(dom.window.document, data); + expect(data).to.have.property('citations'); + const citations = data.citations; + expect(citations).to.be.an.instanceof(Array); + expect(citations).to.have.lengthOf(1); + const citation = citations[0]; + expect(citation).to.equal('test-citation-key'); + }); + it('should extract bibliography', function() { + const dom = new JSDOM(` + sth + + + + `, options); + const data = {}; + const extractBibliography = distill.testing.extractors.get('ExtractBibliography'); + extractBibliography(dom.window.document, data); + expect(data.bibliography).to.be.an.instanceof(Map); + const entry = data.bibliography.get('mercier2011humans'); + expect(entry).to.be.an('object'); + expect(entry).to.have.property('year', '2011'); + }); + + it('should extract front-matter'); + + }); // metadata + + describe('should transform the DOM', function() { + + it('should add Google scholar citation information', function() { + const dom = new JSDOM('', options); + const data = { + authors: [ + {firstName: 'Frank', lastName: 'Underwood', affiliation: 'Google Brain', affiliationURL: 'https://g.co/brain'}, + {firstName: 'Shan', lastName: 'Carter', affiliation: 'Google Brain', affiliationURL: 'https://g.co/brain'}, + ], + doiSuffix: 'test-doi-suffix' + }; + const firstAuthorName = data.authors[0].firstName + ' ' + data.authors[0].lastName; + const GSfirstAuthorName = data.authors[0].lastName + ', ' + data.authors[0].firstName; + + const meta = distill.testing.transforms.get('Meta'); + expect(meta).to.be.a('function'); + + meta(dom.window.document, data); + const metaTags = dom.window.document.querySelectorAll('meta'); + expect(metaTags).to.not.be.empty; + + // Google Scholar + const GSAuthorTags = Array.prototype.filter.call(metaTags, (tag) => { + return tag.name === 'citation_author'; + }); + expect(GSAuthorTags).to.have.lengthOf(2); + const GSFirstAuthorTag = GSAuthorTags[0]; + + expect(GSFirstAuthorTag.content).to.equal(GSfirstAuthorName); + + // Schema.org Author tags + const SOAuthorTags = Array.prototype.filter.call(metaTags, (tag) => { + return tag.getAttribute('property') === 'article:author'; + }); + expect(SOAuthorTags).to.have.lengthOf(2); + const SOFirstAuthorTag = SOAuthorTags[0]; + expect(SOFirstAuthorTag.content).to.equal(firstAuthorName); + + }); + + it('given already correct data, it should add Google scholar references information', function() { + const dom = new JSDOM('', options); + const data = { + doiSuffix: 'test-doi-suffix', + citations: ['test-citation-key'], + bibliography: new Map([[ + 'test-citation-key', { + title: 'Why do humans reason? Arguments for an argumentative theory', + author: 'Mercier, Hugo and Sperber, Dan', + journal: 'Behavioral and brain sciences', + volume: 34, + number: 2 + } + ]]) + }; + const meta = distill.testing.transforms.get('Meta'); + expect(meta).to.be.a('function'); + meta(dom.window.document, data); + const metaTags = [].slice.call(dom.window.document.querySelectorAll('meta[name="citation_reference"]')); + expect(metaTags).to.not.be.empty; + }); + + it('given only a DOM, it should add Google scholar references information', function() { + const dom = new JSDOM(` + sth + + + + `, options); + const data = {}; + distill.render(dom.window.document, data, false); + const metaTags = [].slice.call(dom.window.document.querySelectorAll('meta[name="citation_reference"]')); + expect(metaTags).to.not.be.empty; + }); + + }); + + }); // render it('should export #distillify()', function() { expect(distill.distillify).to.be.a('function'); @@ -61,4 +202,4 @@ describe('Distill V2 (transforms)', function() { }); -}); // describe 'Render' +}); // describe 'Transform'