Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions .eslintrc.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
extends:
- 'wikimedia/server'

plugins:
- json
- jsdoc

rules:
no-shadow:
- off
array-bracket-spacing:
- off
camelcase:
Expand Down
1 change: 1 addition & 0 deletions lib/base_feed.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ class BaseFeed {

/**
* Extract a very header from any of the result parts.
*
* @param {!Object} result
* @return {string|null}
* @private
Expand Down
43 changes: 32 additions & 11 deletions lib/mwUtil.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ const mwUtil = {};
/**
* Create an etag value of the form
* "<revision>/<tid>/<optional_suffix>"
* @param {Integer} rev page revision number
*
* @param {number} rev page revision number
* @param {string} tid page render UUID
* @param {string} [suffix] optional suffix
* @return {string} the value of the ETag header
Expand All @@ -31,6 +32,7 @@ mwUtil.makeETag = (rev, tid, suffix) => {

/**
* Normalizes the request.params.title and returns it back
*
* @param {HyperSwitch} hyper hyperswitch
* @param {Object} req request object
* @param {string} title
Expand All @@ -54,6 +56,7 @@ mwUtil.normalizeTitle = (hyper, req, title) => mwUtil.getSiteInfo(hyper, req)
/**
* Parse an etag value of the form
* "<revision>/<tid>/<suffix>"
*
* @param {string} etag tag to parse
* @return {Object} an object with rev, tid and optional suffix properties
*/
Expand Down Expand Up @@ -85,6 +88,7 @@ mwUtil.parseETag = (etag) => {
/**
* Checks whether a language variant could exist for
* a page in a requested language on a particular wiki.
*
* @param {HyperSwitch} hyper
* @param {Object} req
* @param {string} pageLanguage the language of the requested page.
Expand All @@ -98,6 +102,7 @@ mwUtil.canConvertLangVariant = (hyper, req, pageLanguage) => {
/**
* Checks whether a specialized requested language variant exist for
* a page in a requested language on a particular wiki.
*
* @param {HyperSwitch} hyper
* @param {Object} req
* @param {string} pageLang the language of the requested page.
Expand All @@ -124,6 +129,7 @@ mwUtil.shouldConvertLangVariant = (hyper, req, pageLang, acceptLang) => {
/**
* Extract the date from an `etag` header of the form
* "<revision>/<tid>/<suffix>"
*
* @param {string} etag
* @return {Date|null}
*/
Expand Down Expand Up @@ -155,6 +161,7 @@ mwUtil.coerceTid = (tidString, bucket) => {

/**
* Normalizes the order of 'Content-Type' header fields.
*
* @param {Object} res server response
*/
mwUtil.normalizeContentType = (res) => {
Expand All @@ -170,13 +177,15 @@ mwUtil.normalizeContentType = (res) => {

/**
* Checks whether the request is a 'no-cache' request
*
* @param {Object} req
* @return {boolean}
*/
mwUtil.isNoCacheRequest = (req) => req.headers && /no-cache/i.test(req.headers['cache-control']);

/**
* Checks whether the request is a 'no-store' request
*
* @param {Object} req
* @return {boolean}
*/
Expand Down Expand Up @@ -212,6 +221,7 @@ mwUtil.getLimit = (hyper, req) => {
* if access is restricted, no-op otherwise.
*
* Item might be either a revision or a restriction.
*
* @param {Object} revision the revision object
*/
mwUtil.applyAccessChecks = (revision) => {
Expand Down Expand Up @@ -246,6 +256,7 @@ mwUtil.applyAccessChecks = (revision) => {

/**
* Create a json web token.
*
* @param {HyperSwitch} hyper HyperSwitch context*
* @param {Object} object a JSON object to encode
* @return {string}
Expand All @@ -260,6 +271,7 @@ mwUtil.encodePagingToken = (hyper, object) => {

/**
* Decode signed token and decode the orignal token
*
* @param {HyperSwitch} hyper HyperSwitch context
* @param {string} token paging request token
* @return {Object}
Expand Down Expand Up @@ -304,6 +316,7 @@ mwUtil.getQueryString = (req) => {

/**
* Extracts the domain name from the provided URL.
*
* @param {string} url
* @return {string}
*/
Expand All @@ -325,6 +338,7 @@ mwUtil.addQueryString = (location, query) => {

/**
* Create a `location` header value for a relative redirect.
*
* @param {string} path the path pattern from specInfo.
* @param {hyper.request} req the request
* @param {Object} options with possible parameters:
Expand Down Expand Up @@ -360,10 +374,8 @@ mwUtil.createRelativeTitleRedirect = (path, req, options) => {
mwUtil.getQueryString(req);
} else if (/#/.test(newReqParams.title)) {
const titleFragments = encodeURIComponent(newReqParams.title).split(/%23/);
/* eslint-disable prefer-template */
location = backString + titleFragments.shift() + mwUtil.getQueryString(req) +
'#' + titleFragments.join('%23');
/* eslint-enable prefer-template */
} else {
location = backString + encodeURIComponent(newReqParams.title) + mwUtil.getQueryString(req);
}
Expand All @@ -376,10 +388,11 @@ mwUtil.createRelativeTitleRedirect = (path, req, options) => {
};

/**
* Checks if the request is a CORS request
* @param {Object} req The request to check
* @return {boolean} Whether the request is CORS or not
*/
* Checks if the request is a CORS request
*
* @param {Object} req The request to check
* @return {boolean} Whether the request is CORS or not
*/
mwUtil.isCrossOrigin = (req) => {
return req && req.headers && req.headers.origin && req.headers.origin !== req.params.domain;
};
Expand Down Expand Up @@ -425,6 +438,7 @@ mwUtil.isSelfRedirect = (req, location) => {

/**
* Fetches the summary content from the provided URI
*
* @param {HyperSwitch} hyper HyperSwitch context
* @param {URI} uri summary URI
* @param {headers} headers req header
Expand All @@ -448,6 +462,7 @@ mwUtil.fetchSummary = (hyper, uri, headers) => {
/**
* Traverses through a request body and replaces the given keys with
* the content fetched using the 'fetch' callback
*
* @param {Object} response the response object to hydrate
* @param {Function} fetch the function used to fetch the content if it's not present
* @return {Promise} response
Expand Down Expand Up @@ -494,6 +509,7 @@ mwUtil.hydrateResponse = (response, fetch) => {

/**
* Checks whether the date is today or in the past in UTC-0 timezone
*
* @param {Date} date a date to check
* @return {boolean} true if the date is in the past
*/
Expand All @@ -505,6 +521,7 @@ mwUtil.isHistoric = (date) => {

/**
* Verifies that the date parameter is in proper format.
*
* @param {Object} req the request to check
*/
mwUtil.verifyDateParams = (req) => {
Expand Down Expand Up @@ -543,6 +560,7 @@ mwUtil.verifyDateParams = (req) => {

/**
* Safely builds the Date from request parameters
*
* @param {Object} rp request parameters object
* @return {Date} the requested date.
*/
Expand All @@ -563,6 +581,7 @@ mwUtil.getDateSafe = (rp) => {
/**
* From a list of regexes and strings, constructs a regex that
* matches any of list items
*
* @param {Array.string} list array of strings and regexes
* @return {Object|undefined} either a RegExp object or undefined
*/
Expand All @@ -582,6 +601,7 @@ mwUtil.constructRegex = (list) => {

/**
* Remove entries with duplicate 'title' values from an array of objects
*
* @param {?Array} arr Array of page/article objects
* @param {?Function} upd Optional function to update the original item based on the duplicate.
* Takes two parameters (orig, dupe): orig - the original object,
Expand All @@ -593,9 +613,8 @@ mwUtil.removeDuplicateTitles = (arr, upd) => {
return arr.filter((item, idx, init) => {
if (titles[item.title]) {
if (upd) {
// eslint-disable-next-line no-unused-vars
let orig = init[init.findIndex((el) => el.title === item.title)];
orig = upd(orig, item);
const orig = init[init.findIndex((el) => el.title === item.title)];
upd(orig, item);
}
return false;
}
Expand All @@ -606,13 +625,15 @@ mwUtil.removeDuplicateTitles = (arr, upd) => {

/**
* Checks if the provided path is an HTML route.
*
* @param {string} path the path to check.
* @return {boolean}
*/
mwUtil.isHTMLRoute = (path) => /\/page\/html\//.test(path);

/**
* Extracts the version number from the content-type or accept profile.
*
* @param {string} header the content-type or accept header.
* @return {string|undefined}
*/
Expand Down Expand Up @@ -656,7 +677,7 @@ mwUtil.addVaryHeader = (res, vary) => {

mwUtil.deepCloneObj = (obj) => {
const clonedObj = Array.isArray(obj) ? [] : {};
Object.keys(obj).forEach((key) =>{
Object.keys(obj).forEach((key) => {
const node = obj[key];
clonedObj[key] = (typeof node === 'object' && node !== null) ? mwUtil.deepCloneObj(node) : node;
});
Expand Down
4 changes: 3 additions & 1 deletion lib/security_response_header_filter.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ const cspHeaderVariants = [

/**
* Replaces subdomain with a wildcard
*
* @param {string} domain a full domain name (e.g. en.wikipedia.org)
* @return {string} wildcard version (e.g. *.wikipedia.org)
*/
Expand All @@ -43,7 +44,8 @@ function wildcardSubdomain(domain) {

/**
* Adds Content-Security-Policy & related headers to send in response
* @param {HyperSwitch} hyper the HyperSwitch object
*
* @param {Object} hyper the HyperSwitch object
* @param {Object} req the request to handle
* @param {P} next the promise to execute
* @param {Object} options options containing the following fields:
Expand Down
18 changes: 8 additions & 10 deletions maintenance/generate-wikimedia-domain-config.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@

'use strict';

var preq = require('preq');
var downloadUrl = 'https://en.wikipedia.org/w/api.php?action=sitematrix&format=json';
const preq = require('preq');
const downloadUrl = 'https://en.wikipedia.org/w/api.php?action=sitematrix&format=json';

preq.get({
uri: downloadUrl
})
.then((res) => {
var sm = res.body.sitematrix;
var projects = {
const sm = res.body.sitematrix;
const projects = {
wikipedia: [],
wiktionary: [],
wikiquote: [],
Expand All @@ -28,13 +28,13 @@ preq.get({
};

Object.keys(sm).forEach((k) => {
var lang = sm[k];
const lang = sm[k];
if (lang.site || k === 'specials') {
var sites = lang.site || lang;
const sites = lang.site || lang;
sites.forEach((site) => {
if (site.closed === undefined && site.private === undefined) {
var domain = site.url.replace(/^https?:\/\//, '');
var name = domain.replace(/[^.]+\.(\w+)\.org$/, '$1');
const domain = site.url.replace(/^https?:\/\//, '');
const name = domain.replace(/[^.]+\.(\w+)\.org$/, '$1');
if (projects[name]) {
projects[name].push(domain);
} else {
Expand All @@ -46,10 +46,8 @@ preq.get({
});

Object.keys(projects).forEach((name) => {
// eslint-disable-next-line no-console
console.log(`\n # ${name}`);
projects[name].forEach((domain) => {
// eslint-disable-next-line no-console
console.log(` /{domain:${domain}}: *wp/default/1.0.0`);
});
});
Expand Down
4 changes: 0 additions & 4 deletions maintenance/generate_random_load.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,23 @@ function makeCheck() {
return preq.get(`${testRestBASEUri}/page/html/${encodeURIComponent(title)}/${revision}`)
.then((res) => {
const tid = res.headers.etag.match(/\/(.+)"$/)[1];
// eslint-disable-next-line no-console
console.log(`Testing for ${title}/${revision}/${tid}`);
P.delay(Math.floor(Math.random() * 86400000))
.then(() => {
return P.all([
preq.get(`${testRestBASEUri}/page/html/${encodeURIComponent(title)}/${revision}/${tid}`)
.catch((err) => {
// eslint-disable-next-line no-console
console.log(`${new Date()} Failed to fetch HTML ${title}/${revision}/${tid} from test RB: ${err}`);
}),
preq.get(`${testRestBASEUri}/page/html/${encodeURIComponent(title)}/${revision}/${tid}`)
.catch((err) => {
// eslint-disable-next-line no-console
console.log(`${new Date()} Failed to fetch Data-Parsoid ${title}/${revision}/${tid} from test RB: ${err}`);
})
]);
});

return P.delay(500).then(makeCheck);
}, (e) => {
// eslint-disable-next-line no-console
console.log(`${new Date()} Failed to fetch HTML ${title}/${revision} from test RB: ${e}`);
});
});
Expand Down
Loading