From c9cad8c1a452ce3c66805a40a1fbd1b669bad9c6 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Wed, 19 Oct 2022 15:11:51 +0200 Subject: [PATCH 01/20] send origin server errors back to the frontend --- api/v2/_extract-css-basic.js | 26 ++++++++++++++++++++++++-- api/v2/extract-css.js | 11 ++++++++++- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index 8ca65e1..bfc4f9d 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -80,8 +80,26 @@ export function getStyles(nodes) { return items } +export class HttpError extends Error { + constructor({ url, statusCode, statusText }) { + this.url = url + this.statusCode = statusCode + this.statusText = statusText + this.message = `The origin server at "${url}" errored with status ${statusCode} (${statusText})` + } +} + export async function extractCss(url) { - var { body, headers } = await got(url) + let body = '' + let headers = {} + + try { + var response = await got(url) + body = response.body + headers = response.headers + } catch (error) { + throw new HttpError({ url, statusCode, statusText }) + } // Return early if our response was a CSS file already if (headers['content-type'].includes('text/css')) { @@ -117,7 +135,11 @@ export async function extractCss(url) { // And c'mon, don't @import inside your @import. var importUrls = getImportUrls(item.css) if (importUrls.length > 0) { - var cssRequests = importUrls.map(importUrl => getCssFile(resolveUrl(importUrl, url))) + var cssRequests = importUrls.map( + importUrl => getCssFile(resolveUrl(importUrl, url)) + // silently fail on sub-resources + .catch(_ => '') + ) var importedFiles = await Promise.all(cssRequests) importedFiles.map((css, index) => { result.push({ diff --git a/api/v2/extract-css.js b/api/v2/extract-css.js index 7ceadd9..c49fcb6 100644 --- a/api/v2/extract-css.js +++ b/api/v2/extract-css.js @@ -1,5 +1,5 @@ import { isUrl } from '../_is-url.js' -import { extractCss } from './_extract-css-basic.js' +import { extractCss, HttpError } from './_extract-css-basic.js' export default async (req, res) => { const { url } = req.query @@ -26,6 +26,15 @@ export default async (req, res) => { const css = result.map(({ css }) => css).join('\n') return res.end(css) } catch (error) { + if (error instanceof HttpError) { + res.statusCode = error.statusCode + return res.json({ + url, + statusCode: error.statusCode, + statusText: error.statusText, + message: error.message, + }) + } res.statusCode = 500 return res.json({ message: error.message }) } From 2a9ebf45e42ecdcc74b8cf3540fd98f7326ea0aa Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Wed, 19 Oct 2022 15:17:28 +0200 Subject: [PATCH 02/20] log original error --- api/v2/_extract-css-basic.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index bfc4f9d..b73402c 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -98,6 +98,7 @@ export async function extractCss(url) { body = response.body headers = response.headers } catch (error) { + console.error(error) throw new HttpError({ url, statusCode, statusText }) } @@ -137,8 +138,6 @@ export async function extractCss(url) { if (importUrls.length > 0) { var cssRequests = importUrls.map( importUrl => getCssFile(resolveUrl(importUrl, url)) - // silently fail on sub-resources - .catch(_ => '') ) var importedFiles = await Promise.all(cssRequests) importedFiles.map((css, index) => { From b04db9b377cf5a89b7fe2e44515dd5758bf37aec Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Wed, 19 Oct 2022 15:20:17 +0200 Subject: [PATCH 03/20] pass error text + code --- api/v2/_extract-css-basic.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index b73402c..1d518d9 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -99,7 +99,7 @@ export async function extractCss(url) { headers = response.headers } catch (error) { console.error(error) - throw new HttpError({ url, statusCode, statusText }) + throw new HttpError({ url, statusCode: error.code, statusText: error.message }) } // Return early if our response was a CSS file already From 6c4d28942d1b5b63f23317fb65436f279d66e21d Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Wed, 19 Oct 2022 15:22:28 +0200 Subject: [PATCH 04/20] correctly extend class --- api/v2/_extract-css-basic.js | 1 + 1 file changed, 1 insertion(+) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index 1d518d9..03c25b5 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -82,6 +82,7 @@ export function getStyles(nodes) { export class HttpError extends Error { constructor({ url, statusCode, statusText }) { + super() this.url = url this.statusCode = statusCode this.statusText = statusText From 1c4ea8ff4a96f9b099b4d75528ff3fa7e5f07b34 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Wed, 19 Oct 2022 15:26:28 +0200 Subject: [PATCH 05/20] ditch statusText --- api/v2/_extract-css-basic.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index 03c25b5..a6a5edf 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -81,12 +81,12 @@ export function getStyles(nodes) { } export class HttpError extends Error { - constructor({ url, statusCode, statusText }) { + constructor({ url, statusCode, originalMessage }) { super() this.url = url - this.statusCode = statusCode - this.statusText = statusText - this.message = `The origin server at "${url}" errored with status ${statusCode} (${statusText})` + this.statusCode = statusCode === 'ENOTFOUND' ? 404 : 500 + this.message = `The origin server at "${url}" errored with statusCode ${statusCode}` + this.originalMessage = originalMessage } } @@ -100,7 +100,7 @@ export async function extractCss(url) { headers = response.headers } catch (error) { console.error(error) - throw new HttpError({ url, statusCode: error.code, statusText: error.message }) + throw new HttpError({ url, statusCode: error.code, originalMessage: error.message }) } // Return early if our response was a CSS file already From e5dc24315e74843dea0e2898622ebdabb70aed67 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Wed, 19 Oct 2022 15:29:10 +0200 Subject: [PATCH 06/20] unify statusCode logic --- api/v2/_extract-css-basic.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index a6a5edf..8beab03 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -83,9 +83,11 @@ export function getStyles(nodes) { export class HttpError extends Error { constructor({ url, statusCode, originalMessage }) { super() + + var code = statusCode === 'ENOTFOUND' ? 404 : 500 this.url = url - this.statusCode = statusCode === 'ENOTFOUND' ? 404 : 500 - this.message = `The origin server at "${url}" errored with statusCode ${statusCode}` + this.statusCode = code + this.message = `The origin server at "${url}" errored with statusCode ${code}` this.originalMessage = originalMessage } } From eb8d8a898c8c3e41e7d1656c73d78fb72601259f Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Wed, 19 Oct 2022 15:31:19 +0200 Subject: [PATCH 07/20] add 8s timeouts --- api/v2/_extract-css-basic.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index 8beab03..c2d341b 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -35,7 +35,9 @@ function getImportUrls(css) { export async function getCssFile(url) { try { - var { body } = await got(url) + var { body } = await got(url, { + timeout: 8000 + }) return body } catch (error) { console.error(`CSS not found at ${url} (HTTP ${error.response.statusCode})`) @@ -97,7 +99,9 @@ export async function extractCss(url) { let headers = {} try { - var response = await got(url) + var response = await got(url, { + timeout: 8000 + }) body = response.body headers = response.headers } catch (error) { From acd63e9b9e5b9ce9c39550c3152928fdb90fb035 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Wed, 19 Oct 2022 15:36:46 +0200 Subject: [PATCH 08/20] pass originalMessage --- api/v2/_extract-css-basic.js | 6 +++++- api/v2/extract-css.js | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index c2d341b..977ffb7 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -86,7 +86,11 @@ export class HttpError extends Error { constructor({ url, statusCode, originalMessage }) { super() - var code = statusCode === 'ENOTFOUND' ? 404 : 500 + var code = 500 + if (statusCode === 'ENOTFOUND') { + code = 404 + } + this.url = url this.statusCode = code this.message = `The origin server at "${url}" errored with statusCode ${code}` diff --git a/api/v2/extract-css.js b/api/v2/extract-css.js index c49fcb6..041544f 100644 --- a/api/v2/extract-css.js +++ b/api/v2/extract-css.js @@ -31,11 +31,11 @@ export default async (req, res) => { return res.json({ url, statusCode: error.statusCode, - statusText: error.statusText, message: error.message, + originalMessage: error.originalMessage, }) } res.statusCode = 500 - return res.json({ message: error.message }) + return res.json({ url, message: error.message }) } } From 8ae8aec23bb795b087bf120100b83b4f41a67be1 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 20 Oct 2022 14:29:40 +0200 Subject: [PATCH 09/20] return http 200, remote http code in response --- api/v2/extract-css.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v2/extract-css.js b/api/v2/extract-css.js index 041544f..8c25366 100644 --- a/api/v2/extract-css.js +++ b/api/v2/extract-css.js @@ -27,7 +27,7 @@ export default async (req, res) => { return res.end(css) } catch (error) { if (error instanceof HttpError) { - res.statusCode = error.statusCode + res.statusCode = 200 return res.json({ url, statusCode: error.statusCode, From 5f53a978f698ee544b2c6216ac1eaf7706d53b88 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 20 Oct 2022 14:39:03 +0200 Subject: [PATCH 10/20] log error.response --- api/v2/_extract-css-basic.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index 977ffb7..30ddb91 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -110,6 +110,9 @@ export async function extractCss(url) { headers = response.headers } catch (error) { console.error(error) + if (error.response) { + console.log(error.response) + } throw new HttpError({ url, statusCode: error.code, originalMessage: error.message }) } From 5d5f9b3df5671e8d947cb5cbabab29ec82b3002a Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 20 Oct 2022 14:55:46 +0200 Subject: [PATCH 11/20] respond with original statusCode --- api/v2/_extract-css-basic.js | 9 +++++---- public/index.html | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index 30ddb91..b9a85f4 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -110,10 +110,11 @@ export async function extractCss(url) { headers = response.headers } catch (error) { console.error(error) - if (error.response) { - console.log(error.response) - } - throw new HttpError({ url, statusCode: error.code, originalMessage: error.message }) + throw new HttpError({ + url, + statusCode: error.response ? error.response.statusCode : error.code, + originalMessage: error.message + }) } // Return early if our response was a CSS file already diff --git a/public/index.html b/public/index.html index cedfa0e..a16236d 100644 --- a/public/index.html +++ b/public/index.html @@ -45,14 +45,14 @@

Extract CSS

Example request urls:

From 5cabda432b07b6fc30dcb6fbde62885339177bb5 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 20 Oct 2022 14:58:20 +0200 Subject: [PATCH 12/20] respond with original statusCode --- api/v2/_extract-css-basic.js | 1 + 1 file changed, 1 insertion(+) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index b9a85f4..339aa7f 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -109,6 +109,7 @@ export async function extractCss(url) { body = response.body headers = response.headers } catch (error) { + console.error('status code', error.response?.statusCode) console.error(error) throw new HttpError({ url, From 3fcb0da9c632bb088049015085b108583fdd384e Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 20 Oct 2022 15:01:26 +0200 Subject: [PATCH 13/20] respond with original statusCode --- api/v2/_extract-css-basic.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index 339aa7f..8c43ffb 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -86,14 +86,13 @@ export class HttpError extends Error { constructor({ url, statusCode, originalMessage }) { super() - var code = 500 - if (statusCode === 'ENOTFOUND') { - code = 404 + if (!Number.isFinite(statusCode)) { + statusCode = 500 } this.url = url this.statusCode = code - this.message = `The origin server at "${url}" errored with statusCode ${code}` + this.message = `The origin server at "${url}" errored with statusCode ${statusCode}` this.originalMessage = originalMessage } } From 1c9fda59de77a7f9b8a3f4b2f8a65c0c421a63bf Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 20 Oct 2022 15:49:25 +0200 Subject: [PATCH 14/20] respond with original statusCode --- api/v2/_extract-css-basic.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index 8c43ffb..eb35ecc 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -91,7 +91,7 @@ export class HttpError extends Error { } this.url = url - this.statusCode = code + this.statusCode = statusCode this.message = `The origin server at "${url}" errored with statusCode ${statusCode}` this.originalMessage = originalMessage } From 6532208a0ba7652f9821bb3bdfe8110540de6fdc Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 20 Oct 2022 15:53:14 +0200 Subject: [PATCH 15/20] properly handle 404 --- api/v2/_extract-css-basic.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index eb35ecc..a97f4f8 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -89,6 +89,9 @@ export class HttpError extends Error { if (!Number.isFinite(statusCode)) { statusCode = 500 } + if (statusCode === 'ENOTFOUND') { + statusCode = 404 + } this.url = url this.statusCode = statusCode From 22b34adc31a3c2f0502de3b720e6ef6acc74ead6 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 20 Oct 2022 15:56:46 +0200 Subject: [PATCH 16/20] properly handle 404 --- api/v2/_extract-css-basic.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index a97f4f8..04655b5 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -86,11 +86,10 @@ export class HttpError extends Error { constructor({ url, statusCode, originalMessage }) { super() - if (!Number.isFinite(statusCode)) { - statusCode = 500 - } if (statusCode === 'ENOTFOUND') { statusCode = 404 + } else if (!Number.isFinite(statusCode)) { + statusCode = 500 } this.url = url From 7227330e7c9cdf95a7aeff2e999579444b8f4298 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 27 Oct 2022 19:55:15 +0200 Subject: [PATCH 17/20] stop throwing, use .error instead --- api/v2/_extract-css-basic.js | 39 ++++++++++++++---------------------- api/v2/extract-css.js | 16 ++++++--------- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index 04655b5..fd1a135 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -82,23 +82,6 @@ export function getStyles(nodes) { return items } -export class HttpError extends Error { - constructor({ url, statusCode, originalMessage }) { - super() - - if (statusCode === 'ENOTFOUND') { - statusCode = 404 - } else if (!Number.isFinite(statusCode)) { - statusCode = 500 - } - - this.url = url - this.statusCode = statusCode - this.message = `The origin server at "${url}" errored with statusCode ${statusCode}` - this.originalMessage = originalMessage - } -} - export async function extractCss(url) { let body = '' let headers = {} @@ -110,13 +93,21 @@ export async function extractCss(url) { body = response.body headers = response.headers } catch (error) { - console.error('status code', error.response?.statusCode) - console.error(error) - throw new HttpError({ - url, - statusCode: error.response ? error.response.statusCode : error.code, - originalMessage: error.message - }) + let statusCode = error.response?.statusCode + + if (statusCode === 'ENOTFOUND') { + statusCode = 404 + } else if (!Number.isFinite(statusCode)) { + statusCode = 500 + } + + return { + error: { + statusCode, + message: `The origin server at "${url}" errored with statusCode ${statusCode}`, + originalMessage: error.message, + } + } } // Return early if our response was a CSS file already diff --git a/api/v2/extract-css.js b/api/v2/extract-css.js index 8c25366..61132a1 100644 --- a/api/v2/extract-css.js +++ b/api/v2/extract-css.js @@ -1,5 +1,5 @@ import { isUrl } from '../_is-url.js' -import { extractCss, HttpError } from './_extract-css-basic.js' +import { extractCss } from './_extract-css-basic.js' export default async (req, res) => { const { url } = req.query @@ -15,6 +15,11 @@ export default async (req, res) => { try { const result = await extractCss(url) + if ('error' in result) { + res.statusCode = result.error.statusCode + return res.json(result.error) + } + res.statusCode = 200 res.setHeader('Cache-Control', 'max-age=60') @@ -26,15 +31,6 @@ export default async (req, res) => { const css = result.map(({ css }) => css).join('\n') return res.end(css) } catch (error) { - if (error instanceof HttpError) { - res.statusCode = 200 - return res.json({ - url, - statusCode: error.statusCode, - message: error.message, - originalMessage: error.originalMessage, - }) - } res.statusCode = 500 return res.json({ url, message: error.message }) } From 688a62f46123c788db50896096d275ebb4d8fcf0 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 27 Oct 2022 20:03:25 +0200 Subject: [PATCH 18/20] fix error code detetction --- api/v2/_extract-css-basic.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/v2/_extract-css-basic.js b/api/v2/_extract-css-basic.js index fd1a135..51241d7 100644 --- a/api/v2/_extract-css-basic.js +++ b/api/v2/_extract-css-basic.js @@ -93,11 +93,12 @@ export async function extractCss(url) { body = response.body headers = response.headers } catch (error) { - let statusCode = error.response?.statusCode + let statusCode = error.code - if (statusCode === 'ENOTFOUND') { + if (statusCode === 'ENOTFOUND' || error.message === 'Response code 404 (Not Found)') { statusCode = 404 } else if (!Number.isFinite(statusCode)) { + console.error(error) statusCode = 500 } From 87f9b248059d3db8c16201640bae0302f396482f Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 27 Oct 2022 20:36:37 +0200 Subject: [PATCH 19/20] send http 200 on remote error --- api/v2/extract-css.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/v2/extract-css.js b/api/v2/extract-css.js index 61132a1..cdd0e0f 100644 --- a/api/v2/extract-css.js +++ b/api/v2/extract-css.js @@ -16,11 +16,9 @@ export default async (req, res) => { const result = await extractCss(url) if ('error' in result) { - res.statusCode = result.error.statusCode return res.json(result.error) } - res.statusCode = 200 res.setHeader('Cache-Control', 'max-age=60') if (req.headers.accept.includes('application/json')) { From d6a617d45926d42640db0042b3e3b97bdfb93673 Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 27 Oct 2022 20:37:10 +0200 Subject: [PATCH 20/20] wrap response in error object --- api/v2/extract-css.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v2/extract-css.js b/api/v2/extract-css.js index cdd0e0f..135cb75 100644 --- a/api/v2/extract-css.js +++ b/api/v2/extract-css.js @@ -16,7 +16,7 @@ export default async (req, res) => { const result = await extractCss(url) if ('error' in result) { - return res.json(result.error) + return res.json({ error: result.error }) } res.setHeader('Cache-Control', 'max-age=60')