omnivore/packages/api/test/utils/parser.test.ts

import * as chai from 'chai'
import { expect } from 'chai'
import chaiAsPromised from 'chai-as-promised'
import fs from 'fs'
import 'mocha'
import nock from 'nock'
import { User } from '../../src/entity/user'
import { deleteUser } from '../../src/services/user'
import {
  getTitleFromEmailSubject,
  isProbablyArticle,
  parseEmailAddress,
  parsePageMetadata,
  parsePreparedContent,
} from '../../src/utils/parser'
import { createTestUser } from '../db'

chai.use(chaiAsPromised)

const load = (path: string): string => {
  return fs.readFileSync(path, 'utf8')
}

describe('parseMetadata', () => {
  it('gets author, title, image, description', () => {
    const html = load('./test/utils/data/substack-post.html')
    const metadata = parsePageMetadata(html)
    expect(metadata?.author).to.deep.equal('Omnivore')
    expect(metadata?.title).to.deep.equal('Code Block Syntax Highlighting')
    expect(metadata?.previewImage).to.deep.equal(
      'https://cdn.substack.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F2ab1f7e8-2ca7-4011-8ccb-43d0b3bd244f_1490x2020.png'
    )
    expect(metadata?.description).to.deep.equal(
      'Highlighted <code> in Omnivore'
    )
  })
})

describe('parsePreparedContent', () => {
  it('gets published date when JSONLD fails to load', async () => {
    nock('https://stratechery.com:443', { encodedQueryParams: true })
      .get('/wp-json/oembed/1.0/embed')
      .query({ url: 'https%3A%2F%2Fstratechery.com%2F2016%2Fits-a-tesla%2F' })
      .reply(401)

    const html = load('./test/utils/data/stratechery-blog-post.html')
    const result = await parsePreparedContent('https://blog.omnivore.app/', {
      document: html,
      pageInfo: {},
    })

    expect(result.parsedContent?.publishedDate?.getTime()).to.equal(
      new Date('2016-04-05T15:27:51+00:00').getTime()
    )
  })
  it('returns a highlight range if markers are found in the HTML', async () => {
    const html = `
      <html>
        <body>
          <div>
            <div id='article-container'>
              some prefix text
              <span data-omnivore-highlight-start="true"></span>This is some text within the highlight markers<span data-omnivore-highlight-end="true"></span>
              some suffix text
            </div>
          </div>
        </body>
      </html>
    `
    const result = await parsePreparedContent('https://blog.omnivore.app/', {
      document: html,
      pageInfo: {},
    })

    expect(result.highlightData?.quote).to.eq(
      'This is some text within the highlight markers'
    )
  })
})

// describe('parsePreparedContent', () => {
//   nock('https://oembeddata').get('/').reply(200, {
//     version: '1.0',
//     provider_name: 'Hippocratic Adventures',
//     provider_url: 'https://www.hippocraticadventures.com',
//     title:
//       'The Ultimate Guide to Practicing Medicine in Singapore &#8211; Part 2',
//   })

//   it('gets metadata from external JSONLD if available', async () => {
//     const html = `<html>
//                     <head>
//                     <link rel="alternate" type="application/json+oembed" href="https://oembeddata">
//                     </link
//                     </head>
//                     <body>body</body>
//                     </html>`
//     const result = await parsePreparedContent('https://blog.omnivore.app/', {
//       document: html,
//       pageInfo: {},
//     })
//     expect(result.parsedContent?.title).to.equal(
//       'The Ultimate Guide to Practicing Medicine in Singapore – Part 2'
//     )
//   })
// })

describe('isProbablyArticle', () => {
  let user: User

  before(async () => {
    user = await createTestUser('fakeUser')
  })

  after(async () => {
    await deleteUser(user.id)
  })

  it('returns true when email is signed up with us', async () => {
    const email = user.email
    expect(await isProbablyArticle(email, 'test subject')).to.be.true
  })

  it('returns true when subject has omnivore: prefix', async () => {
    const subject = 'omnivore: test subject'
    expect(await isProbablyArticle('test-email', subject)).to.be.true
  })
})

describe('getTitleFromEmailSubject', () => {
  it('returns the title from the email subject', () => {
    const title = 'test subject'
    const subject = `omnivore: ${title}`
    expect(getTitleFromEmailSubject(subject)).to.eql(title)
  })
})

describe('parseEmailAddress', () => {
  it('returns the name and address when in name <address> format', () => {
    const name = 'test name'
    const address = 'tester@omnivore.app'
    const parsed = parseEmailAddress(`${name} <${address}>`)
    expect(parsed.name).to.eql(name)
    expect(parsed.address).to.eql(address)
  })

  it('returns the address when in address format', () => {
    const address = 'tester@omnivore.app'
    const parsed = parseEmailAddress(address)
    expect(parsed.name).to.eql('')
    expect(parsed.address).to.eql(address)
  })
})
-												Fix tests

											
										
										
											2022-06-01 02:30:23 +00:00
+								import * as chai from 'chai'
-												New function to determine if an HTML blob is probably a newsletter based on its content

											
										
										
											2022-03-03 00:31:15 +00:00
+								import { expect } from 'chai'
-												fix tests

											
										
										
											2024-02-22 04:03:58 +00:00
+								import chaiAsPromised from 'chai-as-promised'
-												New function to determine if an HTML blob is probably a newsletter based on its content

											
										
										
											2022-03-03 00:31:15 +00:00
+								import fs from 'fs'
-												fix tests

											
										
										
											2024-02-22 04:03:58 +00:00
+								import 'mocha'
 								import nock from 'nock'
 								import { User } from '../../src/entity/user'
 								import { deleteUser } from '../../src/services/user'
-												Fix tests

											
										
										
											2022-06-01 02:30:23 +00:00
+								import {
-												Save article from forwarding emails

											
										
										
											2022-07-26 14:08:10 +00:00
+								  getTitleFromEmailSubject,
-												Add function isProbablyArticle to test if a forwarded email contains an article to save

											
										
										
											2022-07-26 10:37:04 +00:00
+								  isProbablyArticle,
-												Add support to the case when from address is in Name <address> format

											
										
										
											2022-07-28 02:26:15 +00:00
+								  parseEmailAddress,
-												Fix tests

											
										
										
											2022-06-01 02:30:23 +00:00
+								  parsePageMetadata,
 								  parsePreparedContent,
 								} from '../../src/utils/parser'
-												fix some rls issue

											
										
										
											2023-09-07 04:35:05 +00:00
+								import { createTestUser } from '../db'
-												Fix tests

											
										
										
											2022-06-01 02:30:23 +00:00
 								chai.use(chaiAsPromised)
-												Attempt to pull URLs for probable newsletter emails out of content

											
										
										
											2022-03-03 07:09:10 +00:00
 								const load = (path: string): string => {
 								  return fs.readFileSync(path, 'utf8')
 								}
-												New function to determine if an HTML blob is probably a newsletter based on its content

											
										
										
											2022-03-03 00:31:15 +00:00
-												Mock sending emails while testing

											
										
										
											2023-03-06 07:57:43 +00:00
+								describe('parseMetadata', () => {
-												Fix some external requests not being mocked

											
										
										
											2023-03-06 08:24:25 +00:00
+								  it('gets author, title, image, description', () => {
-												Fix parsing authors from page metadata

											
										
										
											2022-03-04 03:40:02 +00:00
+								    const html = load('./test/utils/data/substack-post.html')
-												Fix some external requests not being mocked

											
										
										
											2023-03-06 08:24:25 +00:00
+								    const metadata = parsePageMetadata(html)
-												Fix parsing authors from page metadata

											
										
										
											2022-03-04 03:40:02 +00:00
+								    expect(metadata?.author).to.deep.equal('Omnivore')
 								    expect(metadata?.title).to.deep.equal('Code Block Syntax Highlighting')
-												Fix tests

											
										
										
											2022-06-01 02:30:23 +00:00
+								    expect(metadata?.previewImage).to.deep.equal(
 								      'https://cdn.substack.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F2ab1f7e8-2ca7-4011-8ccb-43d0b3bd244f_1490x2020.png'
 								    )
 								    expect(metadata?.description).to.deep.equal(
 								      'Highlighted <code> in Omnivore'
 								    )
-												Fix parsing authors from page metadata

											
										
										
											2022-03-04 03:40:02 +00:00
+								  })
 								})
-												Attempt to set publishedDate if readability fails to parse it

This can happen if JSONLD fails to load. The test page here has
an encoding issue that causes the oembed jsonld to fail to load
and then readability fails to parse the date.

											
										
										
											2022-03-14 04:23:47 +00:00
-												Mock sending emails while testing

											
										
										
											2023-03-06 07:57:43 +00:00
+								describe('parsePreparedContent', () => {
-												Attempt to set publishedDate if readability fails to parse it

This can happen if JSONLD fails to load. The test page here has
an encoding issue that causes the oembed jsonld to fail to load
and then readability fails to parse the date.

											
										
										
											2022-03-14 04:23:47 +00:00
+								  it('gets published date when JSONLD fails to load', async () => {
-												Add highlights to newly saved pages if they contain highlight markers

											
										
										
											2022-11-21 07:54:27 +00:00
+								    nock('https://stratechery.com:443', { encodedQueryParams: true })
-												Add a nock for stratechery test

											
										
										
											2022-11-01 03:51:54 +00:00
+								      .get('/wp-json/oembed/1.0/embed')
-												Add highlights to newly saved pages if they contain highlight markers

											
										
										
											2022-11-21 07:54:27 +00:00
+								      .query({ url: 'https%3A%2F%2Fstratechery.com%2F2016%2Fits-a-tesla%2F' })
-												Add a nock for stratechery test

											
										
										
											2022-11-01 03:51:54 +00:00
+								      .reply(401)
-												Attempt to set publishedDate if readability fails to parse it

This can happen if JSONLD fails to load. The test page here has
an encoding issue that causes the oembed jsonld to fail to load
and then readability fails to parse the date.

											
										
										
											2022-03-14 04:23:47 +00:00
+								    const html = load('./test/utils/data/stratechery-blog-post.html')
-												Fix tests

											
										
										
											2022-10-12 10:05:17 +00:00
+								    const result = await parsePreparedContent('https://blog.omnivore.app/', {
-												Fix tests

											
										
										
											2022-06-01 02:30:23 +00:00
+								      document: html,
 								      pageInfo: {},
 								    })
-												Add a nock for stratechery test

											
										
										
											2022-11-01 03:51:54 +00:00
-												Fix tests

											
										
										
											2022-06-01 02:30:23 +00:00
+								    expect(result.parsedContent?.publishedDate?.getTime()).to.equal(
 								      new Date('2016-04-05T15:27:51+00:00').getTime()
-												Attempt to set publishedDate if readability fails to parse it

This can happen if JSONLD fails to load. The test page here has
an encoding issue that causes the oembed jsonld to fail to load
and then readability fails to parse the date.

											
										
										
											2022-03-14 04:23:47 +00:00
+								    )
 								  })
-												Add highlights to newly saved pages if they contain highlight markers

											
										
										
											2022-11-21 07:54:27 +00:00
+								  it('returns a highlight range if markers are found in the HTML', async () => {
 								    const html = `
 								      <html>
 								        <body>
-												Add prefix and suffix text to make text matching work

											
										
										
											2022-11-21 08:40:17 +00:00
+								          <div>
 								            <div id='article-container'>
 								              some prefix text
 								              <span data-omnivore-highlight-start="true"></span>This is some text within the highlight markers<span data-omnivore-highlight-end="true"></span>
 								              some suffix text
 								            </div>
 								          </div>
-												Add highlights to newly saved pages if they contain highlight markers

											
										
										
											2022-11-21 07:54:27 +00:00
+								        </body>
 								      </html>
 								    `
 								    const result = await parsePreparedContent('https://blog.omnivore.app/', {
 								      document: html,
 								      pageInfo: {},
 								    })
 								    expect(result.highlightData?.quote).to.eq(
 								      'This is some text within the highlight markers'
 								    )
 								  })
-												Use html decoding when getting values from fetched oembed

If we fetch oembed data from an external source, instead of
handling it in readabilityjs we need to html decode it.

											
										
										
											2022-03-16 22:29:39 +00:00
+								})
-												Remove JSONLD test

											
										
										
											2023-10-16 03:39:03 +00:00
+								// describe('parsePreparedContent', () => {
 								//   nock('https://oembeddata').get('/').reply(200, {
 								//     version: '1.0',
 								//     provider_name: 'Hippocratic Adventures',
 								//     provider_url: 'https://www.hippocraticadventures.com',
 								//     title:
 								//       'The Ultimate Guide to Practicing Medicine in Singapore &#8211; Part 2',
 								//   })
-												Use html decoding when getting values from fetched oembed

If we fetch oembed data from an external source, instead of
handling it in readabilityjs we need to html decode it.

											
										
										
											2022-03-16 22:29:39 +00:00
-												Remove JSONLD test

											
										
										
											2023-10-16 03:39:03 +00:00
+								//   it('gets metadata from external JSONLD if available', async () => {
 								//     const html = `<html>
 								//                     <head>
 								//                     <link rel="alternate" type="application/json+oembed" href="https://oembeddata">
 								//                     </link
 								//                     </head>
 								//                     <body>body</body>
 								//                     </html>`
 								//     const result = await parsePreparedContent('https://blog.omnivore.app/', {
 								//       document: html,
 								//       pageInfo: {},
 								//     })
 								//     expect(result.parsedContent?.title).to.equal(
 								//       'The Ultimate Guide to Practicing Medicine in Singapore – Part 2'
 								//     )
 								//   })
 								// })
-												Add function isProbablyArticle to test if a forwarded email contains an article to save

											
										
										
											2022-07-26 10:37:04 +00:00
 								describe('isProbablyArticle', () => {
 								  let user: User
 								  before(async () => {
 								    user = await createTestUser('fakeUser')
 								  })
 								  after(async () => {
-												fix some rls issue

											
										
										
											2023-09-07 04:35:05 +00:00
+								    await deleteUser(user.id)
-												Add function isProbablyArticle to test if a forwarded email contains an article to save

											
										
										
											2022-07-26 10:37:04 +00:00
+								  })
 								  it('returns true when email is signed up with us', async () => {
 								    const email = user.email
 								    expect(await isProbablyArticle(email, 'test subject')).to.be.true
 								  })
 								  it('returns true when subject has omnivore: prefix', async () => {
 								    const subject = 'omnivore: test subject'
 								    expect(await isProbablyArticle('test-email', subject)).to.be.true
 								  })
 								})
-												Save article from forwarding emails

											
										
										
											2022-07-26 14:08:10 +00:00
 								describe('getTitleFromEmailSubject', () => {
 								  it('returns the title from the email subject', () => {
 								    const title = 'test subject'
 								    const subject = `omnivore: ${title}`
 								    expect(getTitleFromEmailSubject(subject)).to.eql(title)
 								  })
 								})
-												Add support to the case when from address is in Name <address> format

											
										
										
											2022-07-28 02:26:15 +00:00
 								describe('parseEmailAddress', () => {
-												Return empty name if name not found in from header

											
										
										
											2022-07-28 02:37:28 +00:00
+								  it('returns the name and address when in name <address> format', () => {
 								    const name = 'test name'
 								    const address = 'tester@omnivore.app'
 								    const parsed = parseEmailAddress(`${name} <${address}>`)
 								    expect(parsed.name).to.eql(name)
 								    expect(parsed.address).to.eql(address)
 								  })
 								  it('returns the address when in address format', () => {
 								    const address = 'tester@omnivore.app'
 								    const parsed = parseEmailAddress(address)
 								    expect(parsed.name).to.eql('')
 								    expect(parsed.address).to.eql(address)
-												Add support to the case when from address is in Name <address> format

											
										
										
											2022-07-28 02:26:15 +00:00
+								  })
 								})