diff --git a/dataops/README.md b/dataops/README.md new file mode 100644 index 000000000..9a417299e --- /dev/null +++ b/dataops/README.md @@ -0,0 +1,3 @@ +```sh +pnpm install +``` \ No newline at end of file diff --git a/dataops/entities.ts b/dataops/entities.ts new file mode 100644 index 000000000..e69de29bb diff --git a/dataops/fetchEntities.ts b/dataops/fetchEntities.ts new file mode 100644 index 000000000..e3ba09eba --- /dev/null +++ b/dataops/fetchEntities.ts @@ -0,0 +1,56 @@ +// TODO: share types +type Entity = any; +/** Query sent to the translation API => load all entitites */ +const entitiesQuery = `query EntitiesQuery { + entities { + id + name + tags + type + category + description + patterns + apiOnly + mdn { + locale + url + title + summary + } + twitterName + twitter { + userName + avatarUrl + } + companyName + company { + name + homepage { + url + } + } + } +} +`; +/** + * Fetch raw entities from the translation API + * @returns + */ +export const fetchEntities = async () => { + const response = await fetch(process.env.TRANSLATION_API!, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ query: entitiesQuery, variables: {} }), + }); + const json = await response.json(); + if (json.errors) { + console.log("// entities API query error"); + console.log(JSON.stringify(json.errors, null, 2)); + throw new Error(); + } + const entities = json?.data?.entities as Array; + return entities; +}; diff --git a/dataops/mongo/connection.ts b/dataops/mongo/connection.ts new file mode 100644 index 000000000..52acdc127 --- /dev/null +++ b/dataops/mongo/connection.ts @@ -0,0 +1,43 @@ +/** + * Mongo db connection with cache, + * works correctly with Next as well + * @see https://github.com/vercel/next.js/blob/canary/examples/with-mongodb/lib/mongodb.ts + * + * @example + * import mongoConnection from "connection" + * const mongoClient = await mongoConnection + * // now you are connected + */ +import { MongoClient } from "mongodb"; + +if (!process.env.MONGODB_URI) { + throw new Error('Invalid environment variable: "MONGODB_URI"'); +} + +const uri = process.env.MONGODB_URI; +const options = {}; + +let client: MongoClient; +let clientPromise: Promise; + +if (!process.env.MONGODB_URI) { + throw new Error("Please add your Mongo URI to .env.local"); +} + +if (process.env.NODE_ENV === "development") { + // In development mode, use a global variable so that the value + // is preserved across module reloads caused by HMR (Hot Module Replacement). + if (!(global as any)._mongoClientPromise) { + client = new MongoClient(uri, options); + (global as any)._mongoClientPromise = client.connect(); + } + clientPromise = (global as any)._mongoClientPromise; +} else { + // In production mode, it's best to not use a global variable. + client = new MongoClient(uri, options); + clientPromise = client.connect(); +} + +// Export a module-scoped MongoClient promise. By doing this in a +// separate module, the client can be shared across functions. +export default clientPromise; diff --git a/dataops/mongo/models.ts b/dataops/mongo/models.ts new file mode 100644 index 000000000..5dcbbacf3 --- /dev/null +++ b/dataops/mongo/models.ts @@ -0,0 +1,9 @@ +import { getCollection } from "./utils"; + +export const getResponseCollection = () => getCollection("responses"); +export const getUserCollection = () => getCollection("users"); +export const getNormalizedResponseCollection = () => + getCollection("normalizedresponses"); + +export const getPrivateResponseCollection = () => + getCollection("privateresponses"); diff --git a/dataops/mongo/utils.ts b/dataops/mongo/utils.ts new file mode 100644 index 000000000..337adba15 --- /dev/null +++ b/dataops/mongo/utils.ts @@ -0,0 +1,17 @@ +import mongoConnection from "./connection"; + +export const connectToAppDb = async () => { + const mongoClient = await mongoConnection; + return mongoClient.db(); +}; + +/** + * Get a collection + * Guarantees that the database is connected before returning + * @param collectionName + * @returns + */ +export const getCollection = async (collectionName: string) => { + const db = await connectToAppDb(); + return db.collection(collectionName); +}; diff --git a/dataops/normalization/countries.ts b/dataops/normalization/countries.ts new file mode 100644 index 000000000..322fb86c4 --- /dev/null +++ b/dataops/normalization/countries.ts @@ -0,0 +1,3242 @@ +///!\ Do not import client-side, would be to big +// TODO: is it up to date in 2022/03? +const countries: Array = [ + { + name: "Afghanistan", + "alpha-2": "AF", + "alpha-3": "AFG", + "country-code": "004", + "iso_3166-2": "ISO 3166-2:AF", + region: "Asia", + "sub-region": "Southern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "034", + "intermediate-region-code": "", + }, + { + name: "Åland Islands", + "alpha-2": "AX", + "alpha-3": "ALA", + "country-code": "248", + "iso_3166-2": "ISO 3166-2:AX", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Albania", + "alpha-2": "AL", + "alpha-3": "ALB", + "country-code": "008", + "iso_3166-2": "ISO 3166-2:AL", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Algeria", + "alpha-2": "DZ", + "alpha-3": "DZA", + "country-code": "012", + "iso_3166-2": "ISO 3166-2:DZ", + region: "Africa", + "sub-region": "Northern Africa", + "intermediate-region": "", + "region-code": "002", + "sub-region-code": "015", + "intermediate-region-code": "", + }, + { + name: "American Samoa", + "alpha-2": "AS", + "alpha-3": "ASM", + "country-code": "016", + "iso_3166-2": "ISO 3166-2:AS", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "Andorra", + "alpha-2": "AD", + "alpha-3": "AND", + "country-code": "020", + "iso_3166-2": "ISO 3166-2:AD", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Angola", + "alpha-2": "AO", + "alpha-3": "AGO", + "country-code": "024", + "iso_3166-2": "ISO 3166-2:AO", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Middle Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "017", + }, + { + name: "Anguilla", + "alpha-2": "AI", + "alpha-3": "AIA", + "country-code": "660", + "iso_3166-2": "ISO 3166-2:AI", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Antarctica", + "alpha-2": "AQ", + "alpha-3": "ATA", + "country-code": "010", + "iso_3166-2": "ISO 3166-2:AQ", + region: "", + "sub-region": "", + "intermediate-region": "", + "region-code": "", + "sub-region-code": "", + "intermediate-region-code": "", + }, + { + name: "Antigua and Barbuda", + "alpha-2": "AG", + "alpha-3": "ATG", + "country-code": "028", + "iso_3166-2": "ISO 3166-2:AG", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Argentina", + "alpha-2": "AR", + "alpha-3": "ARG", + "country-code": "032", + "iso_3166-2": "ISO 3166-2:AR", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Armenia", + "alpha-2": "AM", + "alpha-3": "ARM", + "country-code": "051", + "iso_3166-2": "ISO 3166-2:AM", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Aruba", + "alpha-2": "AW", + "alpha-3": "ABW", + "country-code": "533", + "iso_3166-2": "ISO 3166-2:AW", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Australia", + "alpha-2": "AU", + "alpha-3": "AUS", + "country-code": "036", + "iso_3166-2": "ISO 3166-2:AU", + region: "Oceania", + "sub-region": "Australia and New Zealand", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "053", + "intermediate-region-code": "", + }, + { + name: "Austria", + "alpha-2": "AT", + "alpha-3": "AUT", + "country-code": "040", + "iso_3166-2": "ISO 3166-2:AT", + region: "Europe", + "sub-region": "Western Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "155", + "intermediate-region-code": "", + }, + { + name: "Azerbaijan", + "alpha-2": "AZ", + "alpha-3": "AZE", + "country-code": "031", + "iso_3166-2": "ISO 3166-2:AZ", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Bahamas", + "alpha-2": "BS", + "alpha-3": "BHS", + "country-code": "044", + "iso_3166-2": "ISO 3166-2:BS", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Bahrain", + "alpha-2": "BH", + "alpha-3": "BHR", + "country-code": "048", + "iso_3166-2": "ISO 3166-2:BH", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Bangladesh", + "alpha-2": "BD", + "alpha-3": "BGD", + "country-code": "050", + "iso_3166-2": "ISO 3166-2:BD", + region: "Asia", + "sub-region": "Southern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "034", + "intermediate-region-code": "", + }, + { + name: "Barbados", + "alpha-2": "BB", + "alpha-3": "BRB", + "country-code": "052", + "iso_3166-2": "ISO 3166-2:BB", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Belarus", + "alpha-2": "BY", + "alpha-3": "BLR", + "country-code": "112", + "iso_3166-2": "ISO 3166-2:BY", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "Belgium", + "alpha-2": "BE", + "alpha-3": "BEL", + "country-code": "056", + "iso_3166-2": "ISO 3166-2:BE", + region: "Europe", + "sub-region": "Western Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "155", + "intermediate-region-code": "", + }, + { + name: "Belize", + "alpha-2": "BZ", + "alpha-3": "BLZ", + "country-code": "084", + "iso_3166-2": "ISO 3166-2:BZ", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Central America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "013", + }, + { + name: "Benin", + "alpha-2": "BJ", + "alpha-3": "BEN", + "country-code": "204", + "iso_3166-2": "ISO 3166-2:BJ", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Bermuda", + "alpha-2": "BM", + "alpha-3": "BMU", + "country-code": "060", + "iso_3166-2": "ISO 3166-2:BM", + region: "Americas", + "sub-region": "Northern America", + "intermediate-region": "", + "region-code": "019", + "sub-region-code": "021", + "intermediate-region-code": "", + }, + { + name: "Bhutan", + "alpha-2": "BT", + "alpha-3": "BTN", + "country-code": "064", + "iso_3166-2": "ISO 3166-2:BT", + region: "Asia", + "sub-region": "Southern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "034", + "intermediate-region-code": "", + }, + { + name: "Bolivia (Plurinational State of)", + "alpha-2": "BO", + "alpha-3": "BOL", + "country-code": "068", + "iso_3166-2": "ISO 3166-2:BO", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Bonaire, Sint Eustatius and Saba", + "alpha-2": "BQ", + "alpha-3": "BES", + "country-code": "535", + "iso_3166-2": "ISO 3166-2:BQ", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Bosnia and Herzegovina", + "alpha-2": "BA", + "alpha-3": "BIH", + "country-code": "070", + "iso_3166-2": "ISO 3166-2:BA", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Botswana", + "alpha-2": "BW", + "alpha-3": "BWA", + "country-code": "072", + "iso_3166-2": "ISO 3166-2:BW", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Southern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "018", + }, + { + name: "Bouvet Island", + "alpha-2": "BV", + "alpha-3": "BVT", + "country-code": "074", + "iso_3166-2": "ISO 3166-2:BV", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Brazil", + "alpha-2": "BR", + "alpha-3": "BRA", + "country-code": "076", + "iso_3166-2": "ISO 3166-2:BR", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "British Indian Ocean Territory", + "alpha-2": "IO", + "alpha-3": "IOT", + "country-code": "086", + "iso_3166-2": "ISO 3166-2:IO", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Brunei Darussalam", + "alpha-2": "BN", + "alpha-3": "BRN", + "country-code": "096", + "iso_3166-2": "ISO 3166-2:BN", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Bulgaria", + "alpha-2": "BG", + "alpha-3": "BGR", + "country-code": "100", + "iso_3166-2": "ISO 3166-2:BG", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "Burkina Faso", + "alpha-2": "BF", + "alpha-3": "BFA", + "country-code": "854", + "iso_3166-2": "ISO 3166-2:BF", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Burundi", + "alpha-2": "BI", + "alpha-3": "BDI", + "country-code": "108", + "iso_3166-2": "ISO 3166-2:BI", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Cabo Verde", + "alpha-2": "CV", + "alpha-3": "CPV", + "country-code": "132", + "iso_3166-2": "ISO 3166-2:CV", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Cambodia", + "alpha-2": "KH", + "alpha-3": "KHM", + "country-code": "116", + "iso_3166-2": "ISO 3166-2:KH", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Cameroon", + "alpha-2": "CM", + "alpha-3": "CMR", + "country-code": "120", + "iso_3166-2": "ISO 3166-2:CM", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Middle Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "017", + }, + { + name: "Canada", + "alpha-2": "CA", + "alpha-3": "CAN", + "country-code": "124", + "iso_3166-2": "ISO 3166-2:CA", + region: "Americas", + "sub-region": "Northern America", + "intermediate-region": "", + "region-code": "019", + "sub-region-code": "021", + "intermediate-region-code": "", + }, + { + name: "Cayman Islands", + "alpha-2": "KY", + "alpha-3": "CYM", + "country-code": "136", + "iso_3166-2": "ISO 3166-2:KY", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Central African Republic", + "alpha-2": "CF", + "alpha-3": "CAF", + "country-code": "140", + "iso_3166-2": "ISO 3166-2:CF", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Middle Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "017", + }, + { + name: "Chad", + "alpha-2": "TD", + "alpha-3": "TCD", + "country-code": "148", + "iso_3166-2": "ISO 3166-2:TD", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Middle Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "017", + }, + { + name: "Chile", + "alpha-2": "CL", + "alpha-3": "CHL", + "country-code": "152", + "iso_3166-2": "ISO 3166-2:CL", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "China", + "alpha-2": "CN", + "alpha-3": "CHN", + "country-code": "156", + "iso_3166-2": "ISO 3166-2:CN", + region: "Asia", + "sub-region": "Eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "030", + "intermediate-region-code": "", + }, + { + name: "Christmas Island", + "alpha-2": "CX", + "alpha-3": "CXR", + "country-code": "162", + "iso_3166-2": "ISO 3166-2:CX", + region: "Oceania", + "sub-region": "Australia and New Zealand", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "053", + "intermediate-region-code": "", + }, + { + name: "Cocos (Keeling) Islands", + "alpha-2": "CC", + "alpha-3": "CCK", + "country-code": "166", + "iso_3166-2": "ISO 3166-2:CC", + region: "Oceania", + "sub-region": "Australia and New Zealand", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "053", + "intermediate-region-code": "", + }, + { + name: "Colombia", + "alpha-2": "CO", + "alpha-3": "COL", + "country-code": "170", + "iso_3166-2": "ISO 3166-2:CO", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Comoros", + "alpha-2": "KM", + "alpha-3": "COM", + "country-code": "174", + "iso_3166-2": "ISO 3166-2:KM", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Congo", + "alpha-2": "CG", + "alpha-3": "COG", + "country-code": "178", + "iso_3166-2": "ISO 3166-2:CG", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Middle Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "017", + }, + { + name: "Congo, Democratic Republic of the", + "alpha-2": "CD", + "alpha-3": "COD", + "country-code": "180", + "iso_3166-2": "ISO 3166-2:CD", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Middle Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "017", + }, + { + name: "Cook Islands", + "alpha-2": "CK", + "alpha-3": "COK", + "country-code": "184", + "iso_3166-2": "ISO 3166-2:CK", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "Costa Rica", + "alpha-2": "CR", + "alpha-3": "CRI", + "country-code": "188", + "iso_3166-2": "ISO 3166-2:CR", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Central America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "013", + }, + { + name: "Côte d'Ivoire", + "alpha-2": "CI", + "alpha-3": "CIV", + "country-code": "384", + "iso_3166-2": "ISO 3166-2:CI", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Croatia", + "alpha-2": "HR", + "alpha-3": "HRV", + "country-code": "191", + "iso_3166-2": "ISO 3166-2:HR", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Cuba", + "alpha-2": "CU", + "alpha-3": "CUB", + "country-code": "192", + "iso_3166-2": "ISO 3166-2:CU", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Curaçao", + "alpha-2": "CW", + "alpha-3": "CUW", + "country-code": "531", + "iso_3166-2": "ISO 3166-2:CW", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Cyprus", + "alpha-2": "CY", + "alpha-3": "CYP", + "country-code": "196", + "iso_3166-2": "ISO 3166-2:CY", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Czechia", + "alpha-2": "CZ", + "alpha-3": "CZE", + "country-code": "203", + "iso_3166-2": "ISO 3166-2:CZ", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "Denmark", + "alpha-2": "DK", + "alpha-3": "DNK", + "country-code": "208", + "iso_3166-2": "ISO 3166-2:DK", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Djibouti", + "alpha-2": "DJ", + "alpha-3": "DJI", + "country-code": "262", + "iso_3166-2": "ISO 3166-2:DJ", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Dominica", + "alpha-2": "DM", + "alpha-3": "DMA", + "country-code": "212", + "iso_3166-2": "ISO 3166-2:DM", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Dominican Republic", + "alpha-2": "DO", + "alpha-3": "DOM", + "country-code": "214", + "iso_3166-2": "ISO 3166-2:DO", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Ecuador", + "alpha-2": "EC", + "alpha-3": "ECU", + "country-code": "218", + "iso_3166-2": "ISO 3166-2:EC", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Egypt", + "alpha-2": "EG", + "alpha-3": "EGY", + "country-code": "818", + "iso_3166-2": "ISO 3166-2:EG", + region: "Africa", + "sub-region": "Northern Africa", + "intermediate-region": "", + "region-code": "002", + "sub-region-code": "015", + "intermediate-region-code": "", + }, + { + name: "El Salvador", + "alpha-2": "SV", + "alpha-3": "SLV", + "country-code": "222", + "iso_3166-2": "ISO 3166-2:SV", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Central America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "013", + }, + { + name: "Equatorial Guinea", + "alpha-2": "GQ", + "alpha-3": "GNQ", + "country-code": "226", + "iso_3166-2": "ISO 3166-2:GQ", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Middle Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "017", + }, + { + name: "Eritrea", + "alpha-2": "ER", + "alpha-3": "ERI", + "country-code": "232", + "iso_3166-2": "ISO 3166-2:ER", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Estonia", + "alpha-2": "EE", + "alpha-3": "EST", + "country-code": "233", + "iso_3166-2": "ISO 3166-2:EE", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Eswatini", + "alpha-2": "SZ", + "alpha-3": "SWZ", + "country-code": "748", + "iso_3166-2": "ISO 3166-2:SZ", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Southern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "018", + }, + { + name: "Ethiopia", + "alpha-2": "ET", + "alpha-3": "ETH", + "country-code": "231", + "iso_3166-2": "ISO 3166-2:ET", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Falkland Islands (Malvinas)", + "alpha-2": "FK", + "alpha-3": "FLK", + "country-code": "238", + "iso_3166-2": "ISO 3166-2:FK", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Faroe Islands", + "alpha-2": "FO", + "alpha-3": "FRO", + "country-code": "234", + "iso_3166-2": "ISO 3166-2:FO", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Fiji", + "alpha-2": "FJ", + "alpha-3": "FJI", + "country-code": "242", + "iso_3166-2": "ISO 3166-2:FJ", + region: "Oceania", + "sub-region": "Melanesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "054", + "intermediate-region-code": "", + }, + { + name: "Finland", + "alpha-2": "FI", + "alpha-3": "FIN", + "country-code": "246", + "iso_3166-2": "ISO 3166-2:FI", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "France", + "alpha-2": "FR", + "alpha-3": "FRA", + "country-code": "250", + "iso_3166-2": "ISO 3166-2:FR", + region: "Europe", + "sub-region": "Western Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "155", + "intermediate-region-code": "", + }, + { + name: "French Guiana", + "alpha-2": "GF", + "alpha-3": "GUF", + "country-code": "254", + "iso_3166-2": "ISO 3166-2:GF", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "French Polynesia", + "alpha-2": "PF", + "alpha-3": "PYF", + "country-code": "258", + "iso_3166-2": "ISO 3166-2:PF", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "French Southern Territories", + "alpha-2": "TF", + "alpha-3": "ATF", + "country-code": "260", + "iso_3166-2": "ISO 3166-2:TF", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Gabon", + "alpha-2": "GA", + "alpha-3": "GAB", + "country-code": "266", + "iso_3166-2": "ISO 3166-2:GA", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Middle Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "017", + }, + { + name: "Gambia", + "alpha-2": "GM", + "alpha-3": "GMB", + "country-code": "270", + "iso_3166-2": "ISO 3166-2:GM", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Georgia", + "alpha-2": "GE", + "alpha-3": "GEO", + "country-code": "268", + "iso_3166-2": "ISO 3166-2:GE", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Germany", + "alpha-2": "DE", + "alpha-3": "DEU", + "country-code": "276", + "iso_3166-2": "ISO 3166-2:DE", + region: "Europe", + "sub-region": "Western Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "155", + "intermediate-region-code": "", + }, + { + name: "Ghana", + "alpha-2": "GH", + "alpha-3": "GHA", + "country-code": "288", + "iso_3166-2": "ISO 3166-2:GH", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Gibraltar", + "alpha-2": "GI", + "alpha-3": "GIB", + "country-code": "292", + "iso_3166-2": "ISO 3166-2:GI", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Greece", + "alpha-2": "GR", + "alpha-3": "GRC", + "country-code": "300", + "iso_3166-2": "ISO 3166-2:GR", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Greenland", + "alpha-2": "GL", + "alpha-3": "GRL", + "country-code": "304", + "iso_3166-2": "ISO 3166-2:GL", + region: "Americas", + "sub-region": "Northern America", + "intermediate-region": "", + "region-code": "019", + "sub-region-code": "021", + "intermediate-region-code": "", + }, + { + name: "Grenada", + "alpha-2": "GD", + "alpha-3": "GRD", + "country-code": "308", + "iso_3166-2": "ISO 3166-2:GD", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Guadeloupe", + "alpha-2": "GP", + "alpha-3": "GLP", + "country-code": "312", + "iso_3166-2": "ISO 3166-2:GP", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Guam", + "alpha-2": "GU", + "alpha-3": "GUM", + "country-code": "316", + "iso_3166-2": "ISO 3166-2:GU", + region: "Oceania", + "sub-region": "Micronesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "057", + "intermediate-region-code": "", + }, + { + name: "Guatemala", + "alpha-2": "GT", + "alpha-3": "GTM", + "country-code": "320", + "iso_3166-2": "ISO 3166-2:GT", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Central America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "013", + }, + { + name: "Guernsey", + "alpha-2": "GG", + "alpha-3": "GGY", + "country-code": "831", + "iso_3166-2": "ISO 3166-2:GG", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "Channel Islands", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "830", + }, + { + name: "Guinea", + "alpha-2": "GN", + "alpha-3": "GIN", + "country-code": "324", + "iso_3166-2": "ISO 3166-2:GN", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Guinea-Bissau", + "alpha-2": "GW", + "alpha-3": "GNB", + "country-code": "624", + "iso_3166-2": "ISO 3166-2:GW", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Guyana", + "alpha-2": "GY", + "alpha-3": "GUY", + "country-code": "328", + "iso_3166-2": "ISO 3166-2:GY", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Haiti", + "alpha-2": "HT", + "alpha-3": "HTI", + "country-code": "332", + "iso_3166-2": "ISO 3166-2:HT", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Heard Island and McDonald Islands", + "alpha-2": "HM", + "alpha-3": "HMD", + "country-code": "334", + "iso_3166-2": "ISO 3166-2:HM", + region: "Oceania", + "sub-region": "Australia and New Zealand", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "053", + "intermediate-region-code": "", + }, + { + name: "Holy See", + "alpha-2": "VA", + "alpha-3": "VAT", + "country-code": "336", + "iso_3166-2": "ISO 3166-2:VA", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Honduras", + "alpha-2": "HN", + "alpha-3": "HND", + "country-code": "340", + "iso_3166-2": "ISO 3166-2:HN", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Central America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "013", + }, + { + name: "Hong Kong", + "alpha-2": "HK", + "alpha-3": "HKG", + "country-code": "344", + "iso_3166-2": "ISO 3166-2:HK", + region: "Asia", + "sub-region": "Eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "030", + "intermediate-region-code": "", + }, + { + name: "Hungary", + "alpha-2": "HU", + "alpha-3": "HUN", + "country-code": "348", + "iso_3166-2": "ISO 3166-2:HU", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "Iceland", + "alpha-2": "IS", + "alpha-3": "ISL", + "country-code": "352", + "iso_3166-2": "ISO 3166-2:IS", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "India", + "alpha-2": "IN", + "alpha-3": "IND", + "country-code": "356", + "iso_3166-2": "ISO 3166-2:IN", + region: "Asia", + "sub-region": "Southern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "034", + "intermediate-region-code": "", + }, + { + name: "Indonesia", + "alpha-2": "ID", + "alpha-3": "IDN", + "country-code": "360", + "iso_3166-2": "ISO 3166-2:ID", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Iran (Islamic Republic of)", + "alpha-2": "IR", + "alpha-3": "IRN", + "country-code": "364", + "iso_3166-2": "ISO 3166-2:IR", + region: "Asia", + "sub-region": "Southern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "034", + "intermediate-region-code": "", + }, + { + name: "Iraq", + "alpha-2": "IQ", + "alpha-3": "IRQ", + "country-code": "368", + "iso_3166-2": "ISO 3166-2:IQ", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Ireland", + "alpha-2": "IE", + "alpha-3": "IRL", + "country-code": "372", + "iso_3166-2": "ISO 3166-2:IE", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Isle of Man", + "alpha-2": "IM", + "alpha-3": "IMN", + "country-code": "833", + "iso_3166-2": "ISO 3166-2:IM", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Israel", + "alpha-2": "IL", + "alpha-3": "ISR", + "country-code": "376", + "iso_3166-2": "ISO 3166-2:IL", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Italy", + "alpha-2": "IT", + "alpha-3": "ITA", + "country-code": "380", + "iso_3166-2": "ISO 3166-2:IT", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Jamaica", + "alpha-2": "JM", + "alpha-3": "JAM", + "country-code": "388", + "iso_3166-2": "ISO 3166-2:JM", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Japan", + "alpha-2": "JP", + "alpha-3": "JPN", + "country-code": "392", + "iso_3166-2": "ISO 3166-2:JP", + region: "Asia", + "sub-region": "Eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "030", + "intermediate-region-code": "", + }, + { + name: "Jersey", + "alpha-2": "JE", + "alpha-3": "JEY", + "country-code": "832", + "iso_3166-2": "ISO 3166-2:JE", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "Channel Islands", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "830", + }, + { + name: "Jordan", + "alpha-2": "JO", + "alpha-3": "JOR", + "country-code": "400", + "iso_3166-2": "ISO 3166-2:JO", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Kazakhstan", + "alpha-2": "KZ", + "alpha-3": "KAZ", + "country-code": "398", + "iso_3166-2": "ISO 3166-2:KZ", + region: "Asia", + "sub-region": "Central Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "143", + "intermediate-region-code": "", + }, + { + name: "Kenya", + "alpha-2": "KE", + "alpha-3": "KEN", + "country-code": "404", + "iso_3166-2": "ISO 3166-2:KE", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Kiribati", + "alpha-2": "KI", + "alpha-3": "KIR", + "country-code": "296", + "iso_3166-2": "ISO 3166-2:KI", + region: "Oceania", + "sub-region": "Micronesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "057", + "intermediate-region-code": "", + }, + { + name: "Korea (Democratic People's Republic of)", + "alpha-2": "KP", + "alpha-3": "PRK", + "country-code": "408", + "iso_3166-2": "ISO 3166-2:KP", + region: "Asia", + "sub-region": "Eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "030", + "intermediate-region-code": "", + }, + { + name: "Korea, Republic of", + "alpha-2": "KR", + "alpha-3": "KOR", + "country-code": "410", + "iso_3166-2": "ISO 3166-2:KR", + region: "Asia", + "sub-region": "Eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "030", + "intermediate-region-code": "", + }, + { + name: "Kuwait", + "alpha-2": "KW", + "alpha-3": "KWT", + "country-code": "414", + "iso_3166-2": "ISO 3166-2:KW", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Kyrgyzstan", + "alpha-2": "KG", + "alpha-3": "KGZ", + "country-code": "417", + "iso_3166-2": "ISO 3166-2:KG", + region: "Asia", + "sub-region": "Central Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "143", + "intermediate-region-code": "", + }, + { + name: "Lao People's Democratic Republic", + "alpha-2": "LA", + "alpha-3": "LAO", + "country-code": "418", + "iso_3166-2": "ISO 3166-2:LA", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Latvia", + "alpha-2": "LV", + "alpha-3": "LVA", + "country-code": "428", + "iso_3166-2": "ISO 3166-2:LV", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Lebanon", + "alpha-2": "LB", + "alpha-3": "LBN", + "country-code": "422", + "iso_3166-2": "ISO 3166-2:LB", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Lesotho", + "alpha-2": "LS", + "alpha-3": "LSO", + "country-code": "426", + "iso_3166-2": "ISO 3166-2:LS", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Southern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "018", + }, + { + name: "Liberia", + "alpha-2": "LR", + "alpha-3": "LBR", + "country-code": "430", + "iso_3166-2": "ISO 3166-2:LR", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Libya", + "alpha-2": "LY", + "alpha-3": "LBY", + "country-code": "434", + "iso_3166-2": "ISO 3166-2:LY", + region: "Africa", + "sub-region": "Northern Africa", + "intermediate-region": "", + "region-code": "002", + "sub-region-code": "015", + "intermediate-region-code": "", + }, + { + name: "Liechtenstein", + "alpha-2": "LI", + "alpha-3": "LIE", + "country-code": "438", + "iso_3166-2": "ISO 3166-2:LI", + region: "Europe", + "sub-region": "Western Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "155", + "intermediate-region-code": "", + }, + { + name: "Lithuania", + "alpha-2": "LT", + "alpha-3": "LTU", + "country-code": "440", + "iso_3166-2": "ISO 3166-2:LT", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Luxembourg", + "alpha-2": "LU", + "alpha-3": "LUX", + "country-code": "442", + "iso_3166-2": "ISO 3166-2:LU", + region: "Europe", + "sub-region": "Western Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "155", + "intermediate-region-code": "", + }, + { + name: "Macao", + "alpha-2": "MO", + "alpha-3": "MAC", + "country-code": "446", + "iso_3166-2": "ISO 3166-2:MO", + region: "Asia", + "sub-region": "Eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "030", + "intermediate-region-code": "", + }, + { + name: "Madagascar", + "alpha-2": "MG", + "alpha-3": "MDG", + "country-code": "450", + "iso_3166-2": "ISO 3166-2:MG", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Malawi", + "alpha-2": "MW", + "alpha-3": "MWI", + "country-code": "454", + "iso_3166-2": "ISO 3166-2:MW", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Malaysia", + "alpha-2": "MY", + "alpha-3": "MYS", + "country-code": "458", + "iso_3166-2": "ISO 3166-2:MY", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Maldives", + "alpha-2": "MV", + "alpha-3": "MDV", + "country-code": "462", + "iso_3166-2": "ISO 3166-2:MV", + region: "Asia", + "sub-region": "Southern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "034", + "intermediate-region-code": "", + }, + { + name: "Mali", + "alpha-2": "ML", + "alpha-3": "MLI", + "country-code": "466", + "iso_3166-2": "ISO 3166-2:ML", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Malta", + "alpha-2": "MT", + "alpha-3": "MLT", + "country-code": "470", + "iso_3166-2": "ISO 3166-2:MT", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Marshall Islands", + "alpha-2": "MH", + "alpha-3": "MHL", + "country-code": "584", + "iso_3166-2": "ISO 3166-2:MH", + region: "Oceania", + "sub-region": "Micronesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "057", + "intermediate-region-code": "", + }, + { + name: "Martinique", + "alpha-2": "MQ", + "alpha-3": "MTQ", + "country-code": "474", + "iso_3166-2": "ISO 3166-2:MQ", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Mauritania", + "alpha-2": "MR", + "alpha-3": "MRT", + "country-code": "478", + "iso_3166-2": "ISO 3166-2:MR", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Mauritius", + "alpha-2": "MU", + "alpha-3": "MUS", + "country-code": "480", + "iso_3166-2": "ISO 3166-2:MU", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Mayotte", + "alpha-2": "YT", + "alpha-3": "MYT", + "country-code": "175", + "iso_3166-2": "ISO 3166-2:YT", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Mexico", + "alpha-2": "MX", + "alpha-3": "MEX", + "country-code": "484", + "iso_3166-2": "ISO 3166-2:MX", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Central America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "013", + }, + { + name: "Micronesia (Federated States of)", + "alpha-2": "FM", + "alpha-3": "FSM", + "country-code": "583", + "iso_3166-2": "ISO 3166-2:FM", + region: "Oceania", + "sub-region": "Micronesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "057", + "intermediate-region-code": "", + }, + { + name: "Moldova, Republic of", + "alpha-2": "MD", + "alpha-3": "MDA", + "country-code": "498", + "iso_3166-2": "ISO 3166-2:MD", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "Monaco", + "alpha-2": "MC", + "alpha-3": "MCO", + "country-code": "492", + "iso_3166-2": "ISO 3166-2:MC", + region: "Europe", + "sub-region": "Western Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "155", + "intermediate-region-code": "", + }, + { + name: "Mongolia", + "alpha-2": "MN", + "alpha-3": "MNG", + "country-code": "496", + "iso_3166-2": "ISO 3166-2:MN", + region: "Asia", + "sub-region": "Eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "030", + "intermediate-region-code": "", + }, + { + name: "Montenegro", + "alpha-2": "ME", + "alpha-3": "MNE", + "country-code": "499", + "iso_3166-2": "ISO 3166-2:ME", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Montserrat", + "alpha-2": "MS", + "alpha-3": "MSR", + "country-code": "500", + "iso_3166-2": "ISO 3166-2:MS", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Morocco", + "alpha-2": "MA", + "alpha-3": "MAR", + "country-code": "504", + "iso_3166-2": "ISO 3166-2:MA", + region: "Africa", + "sub-region": "Northern Africa", + "intermediate-region": "", + "region-code": "002", + "sub-region-code": "015", + "intermediate-region-code": "", + }, + { + name: "Mozambique", + "alpha-2": "MZ", + "alpha-3": "MOZ", + "country-code": "508", + "iso_3166-2": "ISO 3166-2:MZ", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Myanmar", + "alpha-2": "MM", + "alpha-3": "MMR", + "country-code": "104", + "iso_3166-2": "ISO 3166-2:MM", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Namibia", + "alpha-2": "NA", + "alpha-3": "NAM", + "country-code": "516", + "iso_3166-2": "ISO 3166-2:NA", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Southern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "018", + }, + { + name: "Nauru", + "alpha-2": "NR", + "alpha-3": "NRU", + "country-code": "520", + "iso_3166-2": "ISO 3166-2:NR", + region: "Oceania", + "sub-region": "Micronesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "057", + "intermediate-region-code": "", + }, + { + name: "Nepal", + "alpha-2": "NP", + "alpha-3": "NPL", + "country-code": "524", + "iso_3166-2": "ISO 3166-2:NP", + region: "Asia", + "sub-region": "Southern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "034", + "intermediate-region-code": "", + }, + { + name: "Netherlands", + "alpha-2": "NL", + "alpha-3": "NLD", + "country-code": "528", + "iso_3166-2": "ISO 3166-2:NL", + region: "Europe", + "sub-region": "Western Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "155", + "intermediate-region-code": "", + }, + { + name: "New Caledonia", + "alpha-2": "NC", + "alpha-3": "NCL", + "country-code": "540", + "iso_3166-2": "ISO 3166-2:NC", + region: "Oceania", + "sub-region": "Melanesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "054", + "intermediate-region-code": "", + }, + { + name: "New Zealand", + "alpha-2": "NZ", + "alpha-3": "NZL", + "country-code": "554", + "iso_3166-2": "ISO 3166-2:NZ", + region: "Oceania", + "sub-region": "Australia and New Zealand", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "053", + "intermediate-region-code": "", + }, + { + name: "Nicaragua", + "alpha-2": "NI", + "alpha-3": "NIC", + "country-code": "558", + "iso_3166-2": "ISO 3166-2:NI", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Central America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "013", + }, + { + name: "Niger", + "alpha-2": "NE", + "alpha-3": "NER", + "country-code": "562", + "iso_3166-2": "ISO 3166-2:NE", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Nigeria", + "alpha-2": "NG", + "alpha-3": "NGA", + "country-code": "566", + "iso_3166-2": "ISO 3166-2:NG", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Niue", + "alpha-2": "NU", + "alpha-3": "NIU", + "country-code": "570", + "iso_3166-2": "ISO 3166-2:NU", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "Norfolk Island", + "alpha-2": "NF", + "alpha-3": "NFK", + "country-code": "574", + "iso_3166-2": "ISO 3166-2:NF", + region: "Oceania", + "sub-region": "Australia and New Zealand", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "053", + "intermediate-region-code": "", + }, + { + name: "North Macedonia", + "alpha-2": "MK", + "alpha-3": "MKD", + "country-code": "807", + "iso_3166-2": "ISO 3166-2:MK", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Northern Mariana Islands", + "alpha-2": "MP", + "alpha-3": "MNP", + "country-code": "580", + "iso_3166-2": "ISO 3166-2:MP", + region: "Oceania", + "sub-region": "Micronesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "057", + "intermediate-region-code": "", + }, + { + name: "Norway", + "alpha-2": "NO", + "alpha-3": "NOR", + "country-code": "578", + "iso_3166-2": "ISO 3166-2:NO", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Oman", + "alpha-2": "OM", + "alpha-3": "OMN", + "country-code": "512", + "iso_3166-2": "ISO 3166-2:OM", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Pakistan", + "alpha-2": "PK", + "alpha-3": "PAK", + "country-code": "586", + "iso_3166-2": "ISO 3166-2:PK", + region: "Asia", + "sub-region": "Southern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "034", + "intermediate-region-code": "", + }, + { + name: "Palau", + "alpha-2": "PW", + "alpha-3": "PLW", + "country-code": "585", + "iso_3166-2": "ISO 3166-2:PW", + region: "Oceania", + "sub-region": "Micronesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "057", + "intermediate-region-code": "", + }, + { + name: "Palestine, State of", + "alpha-2": "PS", + "alpha-3": "PSE", + "country-code": "275", + "iso_3166-2": "ISO 3166-2:PS", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Panama", + "alpha-2": "PA", + "alpha-3": "PAN", + "country-code": "591", + "iso_3166-2": "ISO 3166-2:PA", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Central America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "013", + }, + { + name: "Papua New Guinea", + "alpha-2": "PG", + "alpha-3": "PNG", + "country-code": "598", + "iso_3166-2": "ISO 3166-2:PG", + region: "Oceania", + "sub-region": "Melanesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "054", + "intermediate-region-code": "", + }, + { + name: "Paraguay", + "alpha-2": "PY", + "alpha-3": "PRY", + "country-code": "600", + "iso_3166-2": "ISO 3166-2:PY", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Peru", + "alpha-2": "PE", + "alpha-3": "PER", + "country-code": "604", + "iso_3166-2": "ISO 3166-2:PE", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Philippines", + "alpha-2": "PH", + "alpha-3": "PHL", + "country-code": "608", + "iso_3166-2": "ISO 3166-2:PH", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Pitcairn", + "alpha-2": "PN", + "alpha-3": "PCN", + "country-code": "612", + "iso_3166-2": "ISO 3166-2:PN", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "Poland", + "alpha-2": "PL", + "alpha-3": "POL", + "country-code": "616", + "iso_3166-2": "ISO 3166-2:PL", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "Portugal", + "alpha-2": "PT", + "alpha-3": "PRT", + "country-code": "620", + "iso_3166-2": "ISO 3166-2:PT", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Puerto Rico", + "alpha-2": "PR", + "alpha-3": "PRI", + "country-code": "630", + "iso_3166-2": "ISO 3166-2:PR", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Qatar", + "alpha-2": "QA", + "alpha-3": "QAT", + "country-code": "634", + "iso_3166-2": "ISO 3166-2:QA", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Réunion", + "alpha-2": "RE", + "alpha-3": "REU", + "country-code": "638", + "iso_3166-2": "ISO 3166-2:RE", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Romania", + "alpha-2": "RO", + "alpha-3": "ROU", + "country-code": "642", + "iso_3166-2": "ISO 3166-2:RO", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "Russian Federation", + "alpha-2": "RU", + "alpha-3": "RUS", + "country-code": "643", + "iso_3166-2": "ISO 3166-2:RU", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "Rwanda", + "alpha-2": "RW", + "alpha-3": "RWA", + "country-code": "646", + "iso_3166-2": "ISO 3166-2:RW", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Saint Barthélemy", + "alpha-2": "BL", + "alpha-3": "BLM", + "country-code": "652", + "iso_3166-2": "ISO 3166-2:BL", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Saint Helena, Ascension and Tristan da Cunha", + "alpha-2": "SH", + "alpha-3": "SHN", + "country-code": "654", + "iso_3166-2": "ISO 3166-2:SH", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Saint Kitts and Nevis", + "alpha-2": "KN", + "alpha-3": "KNA", + "country-code": "659", + "iso_3166-2": "ISO 3166-2:KN", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Saint Lucia", + "alpha-2": "LC", + "alpha-3": "LCA", + "country-code": "662", + "iso_3166-2": "ISO 3166-2:LC", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Saint Martin (French part)", + "alpha-2": "MF", + "alpha-3": "MAF", + "country-code": "663", + "iso_3166-2": "ISO 3166-2:MF", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Saint Pierre and Miquelon", + "alpha-2": "PM", + "alpha-3": "SPM", + "country-code": "666", + "iso_3166-2": "ISO 3166-2:PM", + region: "Americas", + "sub-region": "Northern America", + "intermediate-region": "", + "region-code": "019", + "sub-region-code": "021", + "intermediate-region-code": "", + }, + { + name: "Saint Vincent and the Grenadines", + "alpha-2": "VC", + "alpha-3": "VCT", + "country-code": "670", + "iso_3166-2": "ISO 3166-2:VC", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Samoa", + "alpha-2": "WS", + "alpha-3": "WSM", + "country-code": "882", + "iso_3166-2": "ISO 3166-2:WS", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "San Marino", + "alpha-2": "SM", + "alpha-3": "SMR", + "country-code": "674", + "iso_3166-2": "ISO 3166-2:SM", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Sao Tome and Principe", + "alpha-2": "ST", + "alpha-3": "STP", + "country-code": "678", + "iso_3166-2": "ISO 3166-2:ST", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Middle Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "017", + }, + { + name: "Saudi Arabia", + "alpha-2": "SA", + "alpha-3": "SAU", + "country-code": "682", + "iso_3166-2": "ISO 3166-2:SA", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Senegal", + "alpha-2": "SN", + "alpha-3": "SEN", + "country-code": "686", + "iso_3166-2": "ISO 3166-2:SN", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Serbia", + "alpha-2": "RS", + "alpha-3": "SRB", + "country-code": "688", + "iso_3166-2": "ISO 3166-2:RS", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Seychelles", + "alpha-2": "SC", + "alpha-3": "SYC", + "country-code": "690", + "iso_3166-2": "ISO 3166-2:SC", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Sierra Leone", + "alpha-2": "SL", + "alpha-3": "SLE", + "country-code": "694", + "iso_3166-2": "ISO 3166-2:SL", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Singapore", + "alpha-2": "SG", + "alpha-3": "SGP", + "country-code": "702", + "iso_3166-2": "ISO 3166-2:SG", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Sint Maarten (Dutch part)", + "alpha-2": "SX", + "alpha-3": "SXM", + "country-code": "534", + "iso_3166-2": "ISO 3166-2:SX", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Slovakia", + "alpha-2": "SK", + "alpha-3": "SVK", + "country-code": "703", + "iso_3166-2": "ISO 3166-2:SK", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "Slovenia", + "alpha-2": "SI", + "alpha-3": "SVN", + "country-code": "705", + "iso_3166-2": "ISO 3166-2:SI", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Solomon Islands", + "alpha-2": "SB", + "alpha-3": "SLB", + "country-code": "090", + "iso_3166-2": "ISO 3166-2:SB", + region: "Oceania", + "sub-region": "Melanesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "054", + "intermediate-region-code": "", + }, + { + name: "Somalia", + "alpha-2": "SO", + "alpha-3": "SOM", + "country-code": "706", + "iso_3166-2": "ISO 3166-2:SO", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "South Africa", + "alpha-2": "ZA", + "alpha-3": "ZAF", + "country-code": "710", + "iso_3166-2": "ISO 3166-2:ZA", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Southern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "018", + }, + { + name: "South Georgia and the South Sandwich Islands", + "alpha-2": "GS", + "alpha-3": "SGS", + "country-code": "239", + "iso_3166-2": "ISO 3166-2:GS", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "South Sudan", + "alpha-2": "SS", + "alpha-3": "SSD", + "country-code": "728", + "iso_3166-2": "ISO 3166-2:SS", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Spain", + "alpha-2": "ES", + "alpha-3": "ESP", + "country-code": "724", + "iso_3166-2": "ISO 3166-2:ES", + region: "Europe", + "sub-region": "Southern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "039", + "intermediate-region-code": "", + }, + { + name: "Sri Lanka", + "alpha-2": "LK", + "alpha-3": "LKA", + "country-code": "144", + "iso_3166-2": "ISO 3166-2:LK", + region: "Asia", + "sub-region": "Southern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "034", + "intermediate-region-code": "", + }, + { + name: "Sudan", + "alpha-2": "SD", + "alpha-3": "SDN", + "country-code": "729", + "iso_3166-2": "ISO 3166-2:SD", + region: "Africa", + "sub-region": "Northern Africa", + "intermediate-region": "", + "region-code": "002", + "sub-region-code": "015", + "intermediate-region-code": "", + }, + { + name: "Suriname", + "alpha-2": "SR", + "alpha-3": "SUR", + "country-code": "740", + "iso_3166-2": "ISO 3166-2:SR", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Svalbard and Jan Mayen", + "alpha-2": "SJ", + "alpha-3": "SJM", + "country-code": "744", + "iso_3166-2": "ISO 3166-2:SJ", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Sweden", + "alpha-2": "SE", + "alpha-3": "SWE", + "country-code": "752", + "iso_3166-2": "ISO 3166-2:SE", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "Switzerland", + "alpha-2": "CH", + "alpha-3": "CHE", + "country-code": "756", + "iso_3166-2": "ISO 3166-2:CH", + region: "Europe", + "sub-region": "Western Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "155", + "intermediate-region-code": "", + }, + { + name: "Syrian Arab Republic", + "alpha-2": "SY", + "alpha-3": "SYR", + "country-code": "760", + "iso_3166-2": "ISO 3166-2:SY", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Taiwan, Province of China", + "alpha-2": "TW", + "alpha-3": "TWN", + "country-code": "158", + "iso_3166-2": "ISO 3166-2:TW", + region: "Asia", + "sub-region": "Eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "030", + "intermediate-region-code": "", + }, + { + name: "Tajikistan", + "alpha-2": "TJ", + "alpha-3": "TJK", + "country-code": "762", + "iso_3166-2": "ISO 3166-2:TJ", + region: "Asia", + "sub-region": "Central Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "143", + "intermediate-region-code": "", + }, + { + name: "Tanzania, United Republic of", + "alpha-2": "TZ", + "alpha-3": "TZA", + "country-code": "834", + "iso_3166-2": "ISO 3166-2:TZ", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Thailand", + "alpha-2": "TH", + "alpha-3": "THA", + "country-code": "764", + "iso_3166-2": "ISO 3166-2:TH", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Timor-Leste", + "alpha-2": "TL", + "alpha-3": "TLS", + "country-code": "626", + "iso_3166-2": "ISO 3166-2:TL", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Togo", + "alpha-2": "TG", + "alpha-3": "TGO", + "country-code": "768", + "iso_3166-2": "ISO 3166-2:TG", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Western Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "011", + }, + { + name: "Tokelau", + "alpha-2": "TK", + "alpha-3": "TKL", + "country-code": "772", + "iso_3166-2": "ISO 3166-2:TK", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "Tonga", + "alpha-2": "TO", + "alpha-3": "TON", + "country-code": "776", + "iso_3166-2": "ISO 3166-2:TO", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "Trinidad and Tobago", + "alpha-2": "TT", + "alpha-3": "TTO", + "country-code": "780", + "iso_3166-2": "ISO 3166-2:TT", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Tunisia", + "alpha-2": "TN", + "alpha-3": "TUN", + "country-code": "788", + "iso_3166-2": "ISO 3166-2:TN", + region: "Africa", + "sub-region": "Northern Africa", + "intermediate-region": "", + "region-code": "002", + "sub-region-code": "015", + "intermediate-region-code": "", + }, + { + name: "Turkey", + "alpha-2": "TR", + "alpha-3": "TUR", + "country-code": "792", + "iso_3166-2": "ISO 3166-2:TR", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Turkmenistan", + "alpha-2": "TM", + "alpha-3": "TKM", + "country-code": "795", + "iso_3166-2": "ISO 3166-2:TM", + region: "Asia", + "sub-region": "Central Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "143", + "intermediate-region-code": "", + }, + { + name: "Turks and Caicos Islands", + "alpha-2": "TC", + "alpha-3": "TCA", + "country-code": "796", + "iso_3166-2": "ISO 3166-2:TC", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Tuvalu", + "alpha-2": "TV", + "alpha-3": "TUV", + "country-code": "798", + "iso_3166-2": "ISO 3166-2:TV", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "Uganda", + "alpha-2": "UG", + "alpha-3": "UGA", + "country-code": "800", + "iso_3166-2": "ISO 3166-2:UG", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Ukraine", + "alpha-2": "UA", + "alpha-3": "UKR", + "country-code": "804", + "iso_3166-2": "ISO 3166-2:UA", + region: "Europe", + "sub-region": "Eastern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "151", + "intermediate-region-code": "", + }, + { + name: "United Arab Emirates", + "alpha-2": "AE", + "alpha-3": "ARE", + "country-code": "784", + "iso_3166-2": "ISO 3166-2:AE", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "United Kingdom of Great Britain and Northern Ireland", + "alpha-2": "GB", + "alpha-3": "GBR", + "country-code": "826", + "iso_3166-2": "ISO 3166-2:GB", + region: "Europe", + "sub-region": "Northern Europe", + "intermediate-region": "", + "region-code": "150", + "sub-region-code": "154", + "intermediate-region-code": "", + }, + { + name: "United States of America", + "alpha-2": "US", + "alpha-3": "USA", + "country-code": "840", + "iso_3166-2": "ISO 3166-2:US", + region: "Americas", + "sub-region": "Northern America", + "intermediate-region": "", + "region-code": "019", + "sub-region-code": "021", + "intermediate-region-code": "", + }, + { + name: "United States Minor Outlying Islands", + "alpha-2": "UM", + "alpha-3": "UMI", + "country-code": "581", + "iso_3166-2": "ISO 3166-2:UM", + region: "Oceania", + "sub-region": "Micronesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "057", + "intermediate-region-code": "", + }, + { + name: "Uruguay", + "alpha-2": "UY", + "alpha-3": "URY", + "country-code": "858", + "iso_3166-2": "ISO 3166-2:UY", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Uzbekistan", + "alpha-2": "UZ", + "alpha-3": "UZB", + "country-code": "860", + "iso_3166-2": "ISO 3166-2:UZ", + region: "Asia", + "sub-region": "Central Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "143", + "intermediate-region-code": "", + }, + { + name: "Vanuatu", + "alpha-2": "VU", + "alpha-3": "VUT", + "country-code": "548", + "iso_3166-2": "ISO 3166-2:VU", + region: "Oceania", + "sub-region": "Melanesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "054", + "intermediate-region-code": "", + }, + { + name: "Venezuela (Bolivarian Republic of)", + "alpha-2": "VE", + "alpha-3": "VEN", + "country-code": "862", + "iso_3166-2": "ISO 3166-2:VE", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "South America", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "005", + }, + { + name: "Viet Nam", + "alpha-2": "VN", + "alpha-3": "VNM", + "country-code": "704", + "iso_3166-2": "ISO 3166-2:VN", + region: "Asia", + "sub-region": "South-eastern Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "035", + "intermediate-region-code": "", + }, + { + name: "Virgin Islands (British)", + "alpha-2": "VG", + "alpha-3": "VGB", + "country-code": "092", + "iso_3166-2": "ISO 3166-2:VG", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Virgin Islands (U.S.)", + "alpha-2": "VI", + "alpha-3": "VIR", + "country-code": "850", + "iso_3166-2": "ISO 3166-2:VI", + region: "Americas", + "sub-region": "Latin America and the Caribbean", + "intermediate-region": "Caribbean", + "region-code": "019", + "sub-region-code": "419", + "intermediate-region-code": "029", + }, + { + name: "Wallis and Futuna", + "alpha-2": "WF", + "alpha-3": "WLF", + "country-code": "876", + "iso_3166-2": "ISO 3166-2:WF", + region: "Oceania", + "sub-region": "Polynesia", + "intermediate-region": "", + "region-code": "009", + "sub-region-code": "061", + "intermediate-region-code": "", + }, + { + name: "Western Sahara", + "alpha-2": "EH", + "alpha-3": "ESH", + "country-code": "732", + "iso_3166-2": "ISO 3166-2:EH", + region: "Africa", + "sub-region": "Northern Africa", + "intermediate-region": "", + "region-code": "002", + "sub-region-code": "015", + "intermediate-region-code": "", + }, + { + name: "Yemen", + "alpha-2": "YE", + "alpha-3": "YEM", + "country-code": "887", + "iso_3166-2": "ISO 3166-2:YE", + region: "Asia", + "sub-region": "Western Asia", + "intermediate-region": "", + "region-code": "142", + "sub-region-code": "145", + "intermediate-region-code": "", + }, + { + name: "Zambia", + "alpha-2": "ZM", + "alpha-3": "ZMB", + "country-code": "894", + "iso_3166-2": "ISO 3166-2:ZM", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, + { + name: "Zimbabwe", + "alpha-2": "ZW", + "alpha-3": "ZWE", + "country-code": "716", + "iso_3166-2": "ISO 3166-2:ZW", + region: "Africa", + "sub-region": "Sub-Saharan Africa", + "intermediate-region": "Eastern Africa", + "region-code": "002", + "sub-region-code": "202", + "intermediate-region-code": "014", + }, +]; +export default countries; diff --git a/dataops/normalization/cronjob.ts b/dataops/normalization/cronjob.ts new file mode 100644 index 000000000..0ba59247f --- /dev/null +++ b/dataops/normalization/cronjob.ts @@ -0,0 +1,42 @@ +import { ResponseMongooseModel } from "~/modules/responses/model.server"; +import { normalizeResponse } from "./normalize"; + +const limit = 800; + +// every x min, normalize *limit* unnormalized responses +export const normalizeJob = async ({ + entities, + rules, +}: { + entities?: any; + rules?: any; +}) => { + const startAt = new Date(); + const unnormalizedResponses = await ResponseMongooseModel.find( + { + isNormalized: false, + }, + null, + { limit } + ); + const responsesToNormalize = Math.min(unnormalizedResponses.length, limit); + if (unnormalizedResponses.length === 0) { + // eslint-disable-next-line + console.log("// 📊 Found 0 unnormalized responses."); + return; + } + // eslint-disable-next-line + console.log( + `// 📊 Normalizing ${responsesToNormalize}/${unnormalizedResponses.length} unnormalized responses at ${startAt}…` + ); + unnormalizedResponses.forEach(async (response) => { + await normalizeResponse({ document: response, entities, rules }); + }); + const endAt = new Date(); + const diff = Math.abs(endAt.valueOf() - startAt.valueOf()); + const duration = Math.ceil(diff / 1000); + // eslint-disable-next-line + console.log( + `-> 📊 Done normalizing ${responsesToNormalize} responses in ${duration}s` + ); +}; diff --git a/dataops/normalization/helpers.ts b/dataops/normalization/helpers.ts new file mode 100644 index 000000000..f87e4613e --- /dev/null +++ b/dataops/normalization/helpers.ts @@ -0,0 +1,412 @@ +// import { getSetting, runGraphQL, logToFile } from 'meteor/vulcan:core'; +import get from "lodash/get.js"; +import intersection from "lodash/intersection.js"; +import uniqBy from "lodash/uniqBy.js"; +import sortBy from "lodash/sortBy.js"; +import compact from "lodash/compact.js"; +import isEmpty from "lodash/isEmpty.js"; +import { Entity } from "~/modules/entities/typings"; +import { getOrFetchEntities } from "~/modules/entities/server/graphql"; +import { SurveyType, SurveyQuestion } from "~/surveys/typings"; +import { logToFile } from "~/lib/server/debug"; + +/* + +Clean up values to remove 'none', 'n/a', etc. + +*/ +export const ignoreValues = [ + " ", + " ", + " ", + " ", + " ", + " ", + "\n", + "\n\n", + "/", + "\\", + "*", + "+", + "-", + "—", + "n/a", + "N/A", + "NA", + "None", + "none", + "no", + "No", + ".", + "?", +]; +export const cleanupValue = (value) => + typeof value === "undefined" || ignoreValues.includes(value) ? null : value; + +// Global variable that stores the entities +export let entitiesData: { entities?: any; rules?: any } = {}; + +export const getEntitiesData = async () => { + if (isEmpty(entitiesData)) { + entitiesData = await initEntities(); + } + return entitiesData; +}; + +export const initEntities = async () => { + console.log(`// 🗄️ Initializing entities…`); + const entities = await getOrFetchEntities(); + const rules = generateEntityRules(entities); + console.log(` -> Initializing entities done`); + return { entities, rules }; +}; + +/* + +Extract matching tokens from a string + +*/ +const enableLimit = false; +const stringLimit = enableLimit ? 170 : 1000; // max length of string to try and find tokens in +const rulesLimit = 1500; // max number of rules to try and match for any given string +const extractTokens = async ({ value, rules, survey, field, verbose }) => { + const rawString = value; + + if (rawString.length > stringLimit) { + await logToFile( + "normalization_errors.txt", + "Length Error! " + rawString + "\n---\n" + ); + throw new Error( + `Over string limit (${rules.length} rules, ${rawString.length} characters)` + ); + } + + const tokens: Array<{ + id: string; + pattern: string; + match: any; + length: number; + rules: number; + range: [number, number]; + }> = []; + let count = 0; + // extract tokens for each rule, storing + // the start/end index for each match + // to be used later to detect overlap. + for (const { pattern, context, fieldId, id } of rules) { + let scanCompleted = false; + let scanStartIndex = 0; + + // add count to prevent infinite looping + while (scanCompleted !== true && count < rulesLimit) { + count++; + if ( + context && + fieldId && + (context !== survey.context || fieldId !== field.id) + ) { + // if a context and fieldId are defined for the current rule, + // abort unless they match the current context and fieldId + break; + } + const stringToScan = rawString.slice(scanStartIndex); + const match = stringToScan.match(pattern); + if (match !== null) { + const includesToken = !!tokens.find((t) => t.id === id); + if (!includesToken) { + // make sure we don't add an already-matched token more than one time + // for example if someone wrote "React, React, React" + tokens.push({ + id, + pattern: pattern.toString(), + match: match[0], + length: match[0].length, + rules: rules.length, + range: [ + scanStartIndex + match.index, + scanStartIndex + match.index + match[0].length, + ], + }); + } + scanStartIndex += match.index + match[0].length; + } else { + scanCompleted = true; + } + } + } + + // sort by length, longer tokens first + tokens.sort((a, b) => b.length - a.length); + + // for each token look for smaller tokens contained + // in its range and exclude them. + const tokensToExclude: Array = []; + tokens.forEach((token, tokenIndex) => { + // skip already excluded tokens + if (tokensToExclude.includes(tokenIndex)) return; + + tokens.forEach((nestedToken, nestedTokenIndex) => { + // ignore itself & already ignored tokens + if ( + nestedTokenIndex === tokenIndex || + tokensToExclude.includes(nestedTokenIndex) + ) + return; + + // is the nested token contained in the current token range + if ( + nestedToken.range[0] >= token.range[0] && + nestedToken.range[1] <= token.range[1] + ) { + tokensToExclude.push(nestedTokenIndex); + } + }); + }); + + const filteredTokens = tokens.filter( + (token, index) => !tokensToExclude.includes(index) + ); + + const uniqueTokens = uniqBy(filteredTokens, (token) => token.id); + const sortedTokens = sortBy(uniqueTokens, (token) => token.id); + + // ensure ids are uniques + // const uniqueIds = [...new Set(filteredTokens.map((token) => token.id))]; + + // alphabetical sort for consistency + // uniqueIds.sort(); + + return sortedTokens; +}; + +/* + +Normalize a string value + +(Can be limited by tags) + +*/ +export const normalize = async ({ + value, + allRules, + tags, + survey, + field, + verbose, +}: { + value: any; + allRules: Array; + tags?: Array; + survey?: SurveyType; + field?: SurveyQuestion; + verbose?: boolean; +}) => { + const rules = tags + ? allRules.filter((r) => intersection(tags, r.tags).length > 0) + : allRules; + + if (verbose) { + console.log(`// Found ${rules.length} rules to match against`); + } + return await extractTokens({ value, rules, survey, field, verbose }); +}; + +/* + +Normalize a string value and only keep the first result + +*/ +export const normalizeSingle = async (options) => { + const tokens = await normalize(options); + // put longer tokens first as a proxy for relevancy + const sortedTokens = sortBy(tokens, (v) => v.id && v.id.length).reverse(); + return sortedTokens[0]; +}; + +/* + +Handle source normalization separately since its value can come from +three different fields (source field, referrer field, 'how did you hear' field) + +*/ +export const normalizeSource = async (normResp, allRules, survey) => { + const tags = [ + "sites", + "podcasts", + "youtube", + "socialmedia", + "newsletters", + "people", + "sources", + ]; + + // add a special rule for emails to normalize "email" sources into current survey id + const allRulesWithEmail = [ + ...allRules, + { + id: survey.normalizationId, + pattern: /e( |-)*mail/i, + tags: ["sources"], + }, + { + id: survey.normalizationId, + pattern: /mail.google.com/i, + tags: ["sources"], + }, + ]; + + const rawSource = get(normResp, "user_info.sourcetag"); + const rawFindOut = get( + normResp, + "user_info.how_did_user_find_out_about_the_survey" + ); + const rawRef = get(normResp, "user_info.referrer"); + + try { + const normSource = + rawSource && + (await normalizeSingle({ + value: rawSource, + allRules, + tags, + survey, + field: { id: "source" }, + })); + const normFindOut = + rawFindOut && + (await normalizeSingle({ + value: rawFindOut, + allRules: allRulesWithEmail, + tags, + survey, + field: { id: "how_did_user_find_out_about_the_survey" }, + })); + const normReferrer = + rawRef && + (await normalizeSingle({ + value: rawRef, + allRules: allRulesWithEmail, + tags, + survey, + field: { id: "referrer" }, + })); + + if (normSource) { + return { ...normSource, raw: rawSource }; + } else if (normFindOut) { + return { ...normFindOut, raw: rawFindOut }; + } else if (normReferrer) { + return { ...normReferrer, raw: rawRef }; + } else { + return { raw: compact([rawSource, rawFindOut, rawRef]).join(", ") }; + } + } catch (error) { + console.log( + `// normaliseSource error for response ${normResp.responseId} with values ${rawSource}, ${rawFindOut}, ${rawRef}` + ); + throw new Error(error); + } +}; + +/* + +Generate normalization rules from entities + +*/ +export const generateEntityRules = (entities: Array) => { + const rules: Array<{ + id: string; + pattern: RegExp | string; + context?: string; + fieldId?: string; + tags: Array; + }> = []; + entities + .filter((e) => !e.apiOnly) + .forEach((entity) => { + const { id, patterns, tags, twitterName } = entity; + // we match the separator group 0 to 2 times to account for double spaces, + // double hyphens, etc. + const separator = "( |-|_|.){0,2}"; + + // 1. replace "_" by separator + const idPatternString = id.replaceAll("_", separator); + const idPattern = new RegExp(idPatternString, "i"); + rules.push({ + id, + pattern: idPattern, + tags, + }); + + // note: the following should not be needed because all entities + // should already follow the foo_js/foo_css forma + // 2. replace "js" at the end by separator+js + //if (id.substr(-2) === "js") { + // const patternString = id.substr(0, id.length - 2) + separator + "js"; + // const pattern = new RegExp(patternString, "i"); + // rules.push({ id, pattern, tags }); + //} + // + // // 3. replace "css" at the end by separator+css + // if (id.substr(-3) === "css") { + // const patternString = id.substr(0, id.length - 3) + separator + "css"; + // const pattern = new RegExp(patternString, "i"); + // rules.push({ id, pattern, tags }); + // } + + // 4. add custom patterns + patterns && + patterns.forEach((patternString) => { + /* + Some patterns are of the form context||question||pattern + to only match a specific question in a specific context. + + For example, for entity "graphql_org": + + state_of_graphql||sites_courses||official documentation + */ + + const patternSegments = patternString.split("||"); + if (patternSegments.length === 3) { + const pattern = new RegExp(patternSegments[2], "i"); + rules.push({ + id, + pattern, + context: patternSegments[0], + fieldId: patternSegments[1], + tags, + }); + } else { + const pattern = new RegExp(patternSegments[0], "i"); + rules.push({ id, pattern, tags }); + } + }); + + // 5. also add twitter username if available (useful for people entities) + if (twitterName) { + const pattern = new RegExp(twitterName, "i"); + rules.push({ id, pattern, tags }); + } + }); + return rules; +}; + +export const logAllRules = async () => { + const allEntities = await getOrFetchEntities(); + if (allEntities) { + let rules = generateEntityRules(allEntities); + rules = rules.map(({ id, pattern, tags }) => ({ + id, + pattern: pattern.toString(), + tags, + })); + const json = JSON.stringify(rules, null, 2); + + await logToFile("rules.js", "export default " + json, { + mode: "overwrite", + }); + } else { + console.log("// Could not get entities"); + } +}; diff --git a/dataops/normalization/normalize.ts b/dataops/normalization/normalize.ts new file mode 100644 index 000000000..a996acdc1 --- /dev/null +++ b/dataops/normalization/normalize.ts @@ -0,0 +1,447 @@ +// TODO: should be imported dynamically +import countries from "./countries"; +import { + cleanupValue, + generateEntityRules, + normalize, + normalizeSource, +} from "./helpers"; +import { getOrFetchEntities } from "~/modules/entities/server/graphql"; +import set from "lodash/set.js"; +import last from "lodash/last.js"; +import intersection from "lodash/intersection.js"; +//import NormalizedResponses from "~/modules/normalized_responses/collection"; +//import Responses from "~/modules/responses/collection"; +//import PrivateResponses from "~/modules/private_responses/collection"; + +// TODO: we need to import the list of surveys from a shared folder as well +import { getSurveyBySlug } from "~/modules/surveys/helpers"; +import isEmpty from "lodash/isEmpty.js"; +import { Field, SurveyType } from "~/surveys"; +//import { +// NormalizedResponseDocument, +// NormalizedResponseMongooseModel, +//} from "~/admin/models/normalized_responses/model.server"; +// TODO share types +type NormalizedResponseDocument = any; +import { ResponseMongooseModel } from "~/modules/responses/model.server"; +//import { +// PrivateResponseDocument, +// PrivateResponseMongooseModel, +//} from "~/admin/models/private_responses/model.server"; +type PrivateResponseDocument = any; +import { getUUID } from "~/account/email/api/encryptEmail"; +import { logToFile } from "~/lib/server/debug"; +import { + getNormalizedResponseCollection, + getPrivateResponseCollection, +} from "../mongo/models"; + +// import { ObjectId } from "mongo"; + +const replaceAll = function ( + target: string, + search: string, + replacement: string, +) { + return target.replace(new RegExp(search, "g"), replacement); +}; + +const convertForCSV = (obj: any) => { + if (!obj || (Array.isArray(obj) && obj.length === 0)) { + return ""; + } else if (typeof obj === "string") { + return obj; + } else { + let s = JSON.stringify(obj); + s = replaceAll(s, '"', `'`); + // s = replaceAll(s, ',', '\,'); + return s; + } +}; + +const logRow = async (columns: Array, fileName: string) => { + await logToFile( + `${fileName}.csv`, + columns.map((c) => `"${convertForCSV(c)}"`).join(", "), + ); +}; + +// fields to copy, along with the path at which to copy them (if different) +const fieldsToCopy = [ + ["surveySlug"], + ["createdAt"], + ["updatedAt"], + ["finishedAt"], + ["completion"], + ["userId"], + ["isFake"], + ["isFinished"], + ["knowledgeScore", "user_info.knowledge_score"], + ["common__user_info__device", "user_info.device"], + ["common__user_info__browser", "user_info.browser"], + ["common__user_info__version", "user_info.version"], + ["common__user_info__os", "user_info.os"], + ["common__user_info__referrer", "user_info.referrer"], + ["common__user_info__source", "user_info.sourcetag"], + ["common__user_info__authmode", "user_info.authmode"], +]; + +// a response must have at least one of those fields to be added to the normalized dataset +// (discard empty responses) +const mustHaveKeys = [ + "features", + "tools", + "resources", + "usage", + "opinions", + "environments", +]; + +const privateFieldPaths = [ + "user_info.github_username", + "user_info.twitter_username", +]; + +interface NormalizedField { + fieldName?: string; + value: any; + normTokens?: Array; +} +export const normalizeResponse = async ({ + document: response, + entities, + rules, + log = false, + fileName: _fileName, + verbose = false, +}: { + document: any; + entities?: Array; + rules?: any; + log?: Boolean; + fileName?: string; + verbose?: boolean; +}): Promise< + | { + result: { _id: string }; + normalizedFields: Array; + } + | undefined +> => { + try { + if (verbose) { + console.log(`// Normalizing document ${response._id}…`); + } + + const normResp: Partial = {}; + const privateFields: any = {}; + const normalizedFields: Array = []; + const survey = getSurveyBySlug(response.surveySlug); + if (!survey) { + throw new Error(`Could not find survey for slug ${response.surveySlug}`); + } + + let allEntities; + if (entities) { + allEntities = entities; + } else { + console.log("// Getting/fetching entities…"); + allEntities = await getOrFetchEntities(); + } + + const allRules = rules ?? generateEntityRules(allEntities); + const fileName = _fileName || `${response.surveySlug}_normalization`; + + /* + + 1. Copy over root fields and assign id + + */ + fieldsToCopy.forEach((field) => { + const [fieldName, fieldPath = fieldName] = field; + set(normResp, fieldPath, response[fieldName]); + }); + normResp.responseId = response._id; + normResp.generatedAt = new Date(); + normResp.survey = survey.context; + normResp.year = survey.year; + + /* + + 2. Generate email hash + + TODO: clarifiy, is this the email from current user (we + don't have it anymore), or the email from "user_info" part, + which will stay and should be hashed? + + NOTE: eventhough we don't store the user email, + we can have an "email" field in the survey if user still + want to send their email afterward + => we need to hash it as well + + */ + if (response.emailHash) { + // If we have already seen this email, use the same uuid + // Otherwise create a new one + const emailHash = response.emailHash; + const emailUuid = await getUUID(emailHash, response.userId); + set(normResp, "user_info.uuid", emailUuid); + } + + /* + + 3. Store locale + + Note: change 'en', 'en-GB', 'en-AU', etc. to 'en-US' for consistency + + */ + const enLocales = ["en", "en-GB", "en-CA", "en-AU", "en,en"]; + const locale = enLocales.includes(response.locale) + ? "en-US" + : response.locale; + set(normResp, "user_info.locale", locale); + + /* + + 4. Loop over survey sections and fields (a.k.a. questions) + + */ + for (const s of survey.outline) { + for (const field of s.questions) { + const { fieldName, matchTags = [] } = field as Field; + if (!fieldName) throw new Error(`Field without fieldName`); + + const [initialSegment, ...restOfPath] = fieldName.split("__"); + const normPath = restOfPath.join("."); + const value = response[fieldName]; + // clean value to eliminate empty spaces, "none", "n/a", etc. + const cleanValue = cleanupValue(value); + + if (cleanValue !== null) { + if (privateFieldPaths.includes(normPath)) { + // handle private info fields separately + set(privateFields, normPath, value); + } else { + if (last(restOfPath) === "others") { + // A. "others" fields needing to be normalized + set(normResp, `${normPath}.raw`, value); + + if (log) { + await logToFile( + `${fileName}.txt`, + `${response._id}, ${fieldName}, ${cleanValue}, ${matchTags.toString()}`, + ); + } + try { + if (verbose) { + console.log( + `// Normalizing key "${fieldName}" with value "${value}" and tags ${matchTags.toString()}…`, + ); + } + + const normTokens = await normalize({ + value: cleanValue, + allRules, + tags: matchTags, + survey, + field, + verbose, + }); + if (verbose) { + console.log( + ` -> Normalized values: ${JSON.stringify(normTokens)}`, + ); + } + + // console.log( + // ` -> Normalized values: ${JSON.stringify(normTokens)}` + // ); + + if (log) { + if (normTokens.length > 0) { + normTokens.forEach(async (token: any) => { + const { id, pattern, rules, match } = token; + await logRow( + [ + response._id, + fieldName, + value, + matchTags, + id, + pattern, + rules, + match, + ], + fileName, + ); + }); + } else { + await logRow( + [ + response._id, + fieldName, + value, + matchTags, + "n/a", + "n/a", + "n/a", + "n/a", + ], + fileName, + ); + } + } + + const normIds = normTokens.map((token: any) => token.id); + const normPatterns = normTokens.map((token: any) => + token.pattern.toString() + ); + set(normResp, `${normPath}.normalized`, normIds); + set(normResp, `${normPath}.patterns`, normPatterns); + + // keep trace of fields that were normalized + normalizedFields.push({ + fieldName, + value, + normTokens, + }); + } catch (error: any) { + set(normResp, `${normPath}.error`, error.message); + } + } else if (last(restOfPath) === "prenormalized") { + // B. these fields are "prenormalized" through autocomplete inputs + const newPath = normPath.replace(".prenormalized", ".others"); + set(normResp, `${newPath}.raw`, value); + set(normResp, `${newPath}.normalized`, value); + set(normResp, `${newPath}.patterns`, ["prenormalized"]); + } else { + // C. any other field + set(normResp, normPath, value); + } + } + } + } + } + + /* + + 5. Discard any "empty" responses + + */ + if (intersection(Object.keys(normResp), mustHaveKeys).length === 0) { + if (verbose) { + console.log(`!! Discarding response ${response._id} as empty`); + } + return; + } + + /* + + 6. Normalize country (if provided) + + */ + if (normResp?.user_info?.country) { + set(normResp, "user_info.country_alpha2", normResp.user_info.country); + const countryNormalized = countries.find( + (c) => c["alpha-2"] === normResp?.user_info?.country, + ); + if (countryNormalized) { + set(normResp, "user_info.country_name", countryNormalized.name); + set(normResp, "user_info.country_alpha3", countryNormalized["alpha-3"]); + } else { + if (log) { + await logToFile( + "countries_normalization.txt", + normResp.user_info.country, + ); + } + } + } + + /* + + 7. Handle source field separately + + */ + const normSource = await normalizeSource(normResp, allRules, survey); + if (normSource.raw) { + set(normResp, "user_info.source.raw", normSource.raw); + } + if (normSource.id) { + set(normResp, "user_info.source.normalized", normSource.id); + } + if (normSource.pattern) { + set(normResp, "user_info.source.pattern", normSource.pattern.toString()); + } + + /* + + 8. Store identifying info in a separate collection + + */ + if (!isEmpty(privateFields)) { + const privateInfo: + & Partial + & Pick = { + user_info: {}, + ...privateFields, + surveySlug: response.surveySlug, + responseId: response._id, + }; + + const PrivateResponseCollection = await getPrivateResponseCollection(); + // NOTE: findOneAndUpdate and updateOne with option "upsert:true" are roughly equivalent, + // but update is probably faster when appliable (the result will have a different shape) + await PrivateResponseCollection.updateOne( + { responseId: response._id }, + privateInfo, + { upsert: true }, + ); + //set(normResp, "user_info.hash", createHash(response.email)); + } + + // console.log(JSON.stringify(normResp, '', 2)); + + // explicitely create string _id + // doesn't work currently: + // MongoServerError: Performing an update on the path '_id' would modify the immutable field '_id' + // normResp._id = (new ObjectId()).toString() + + // update normalized response, or insert it if it doesn't exist + // NOTE: this will generate ObjectId _id for unknown reason, see https://github.com/Devographics/StateOfJS-next2/issues/31 + const NormalizedReponseCollection = await getNormalizedResponseCollection(); + const updatedNormalizedResponseRes = await NormalizedReponseCollection + .findOneAndUpdate( + { responseId: response._id }, + normResp, + { upsert: true, returnDocument: "after" }, + ); + const updatedNormalizedResponse = updatedNormalizedResponseRes.value!; + await ResponseMongooseModel.updateOne( + { _id: response._id }, + { + $set: { + normalizedResponseId: updatedNormalizedResponse._id, + isNormalized: true, + }, + }, + ); + + // eslint-disable-next-line + // console.log(result); + return { + /* + Previously was the result of "upsert", but in Mongoose we use findOneAndUpdate instead + result,*/ + + // TODO: we have to be cautious with string ids... + // @ts-expect-error + result: updatedNormalizedResponse, + normalizedFields, + }; + } catch (error) { + console.log("// normalizeResponse error"); + console.log(error); + } +}; diff --git a/dataops/normalization/rules.js b/dataops/normalization/rules.js new file mode 100644 index 000000000..c41d14856 --- /dev/null +++ b/dataops/normalization/rules.js @@ -0,0 +1,505 @@ +// TODO: seems unused +export const resourceNormalizationRules = [ + [/bitsofco\.de/i, "bitsofcode"], + [/web tools weekly/i, "web_tools_weekly"], + [/web design weekly/i, "web_design_weekly"], + [/vue revue/i, "vue_revue"], + [/ember ?weekly/i, "ember_weekly"], + [/elm ?weekly/i, "elm_weekly"], + [/rascia/i, "tania_rascia"], + [/react ?status/i, "react_status"], + [/angular ?in ?depth/i, "angular_in_depth"], + + // podcasts + + [/react ?podcast/i, "react_podcast"], + [/putaindecode/i, "putaindecode"], + [/if\/else/i, "if_else_podcast"], + [/devmode/i, "devmode"], + [/dev ?schacht/i, "dev_schacht"], + [/corecursive/i, "corecursive"], + [/coding ?blocks/i, "coding_blocks"], + [/working ?draft/i, "working_draft"], + [/webbidevaus/i, "webbidevaus"], + [/views ?on ?vue/i, "views_on_vue"], + [/bikeshed/i, "bikeshed"], + [/react ?podcast/i, "react_podcast"], + [/talkscript/i, "talkscript"], + [/soft ?skills ?engineering/i, "soft_skills_engineering"], + [/reason ?town/i, "reason_town"], + [/real ?talk/i, "real_talk_javascript"], + [/react ?native ?radio/i, "react_native_radio"], + [/hipsters/i, "hipsters_tech"], + [/functional ?geekery/i, "functional_geekery"], + [/frontend ?weekend/i, "frontend_weekend"], + [/embermap/i, "embermap"], + [/base ?cs/i, "base_cs"], + [/angular ?air/i, "angular_air"], + [/adventures ?in ?angular/i, "adventures_in_angular"], + [/net ?rocks/i, "dot_net_rocks"], + [/a11y ?rules/, "a11y_rules"], + [/bike ?shed/, "the_bike_shed"], + + [/schwarzmuller/i, "schwarzmuller"], + [/schwarzmüller/i, "schwarzmuller"], + [/shwarzmüller/i, "schwarzmuller"], + + [/youtube/i, "youtube"], + [/vue ?school/i, "vueschool"], + [/laracasts/i, "laracasts"], + [/vue ?mastery/i, "vuemastery"], + [/ultimate ?course/i, "ultimate_courses"], + [/udemy/i, "udemy"], + [/platzi/i, "platzi"], + [/udacity/i, "udacity"], + [/treehouse/i, "treehouse"], + [/testingjavascript/i, "testingjavascript"], + [/scrimba/i, "scrimba"], + [/packtpub/i, "packtpub"], + [/javascript\.info/i, "javascript_info"], + [/front ?end ?masters/i, "frontendmasters"], + [/exercism/i, "exercism"], + [/devdocs/i, "devdocs"], + [/codewars/i, "codewars"], + [/caniuse/i, "caniuse"], + [/lynda/i, "lynda"], + [/traversy/i, "traversy"], + [/odin/i, "odin_project"], + [/open ?class ?room/i, "openclassroom"], + [/linkedin ?learning/i, "linkedin_learning"], + [/fireship/i, "fireship"], + [/academind/i, "academind"], + + [/enjoy ?the ?vue/i, "enjoy_the_vue"], +]; + +/** + * Defines a set of rules which can be applied + * in order to standardize tool names, it's mostly involved + * to extract things from the "other tools" questions. + * + * ⚠️ ORDER MATTERS + */ +export const toolNormalizationRules = [ + [ + /(Good Old Plain JavaScript|"Plain" JavaScript \(ES5\)|vanilla)/i, + "vanillajs", + ], + [/es6/i, "es6"], + [/coffescript/i, "coffeescript"], + [/coffeescript/i, "coffeescript"], + [/typescript/i, "typescript"], + [/flow/i, "flow"], + [/elm-test/i, "elm_test"], + [/elm( |-)?native/i, "elm_native"], + [/elm/i, "elm"], + [/reason/i, "reason"], + [/clojure/i, "clojurescript"], + [/No Front-End Framework/i, "nofrontendframework"], + [/preact/i, "preact"], + [/^react$/i, "react"], + [/aurelia/i, "aurelia"], + [/polymer/i, "polymer"], + [/^angular$/i, "angular"], + [/angular ?2/i, "angular"], + [/angular ?1/i, "angular_1"], + [/angular( |-)?native/i, "angular_native"], + [/ember/i, "ember"], + [/ember( |-|\.)?data/i, "ember_data"], + [/^vue(\.js)?$/i, "vuejs"], + [/backbone/i, "backbone"], + [/redux/i, "redux"], + [/mobx/i, "mobx"], + [/rest( |-)?api/i, "rest"], + [/restify/i, "restify"], + [/firebase/i, "firebase"], + [/graphql/i, "graphql"], + [/apollo/i, "apollo"], + [/falcor/i, "falcor"], + [/horizon/i, "horizon"], + [/(meteor|blaze)/i, "meteor"], + [/feathers/i, "feathers"], + [/donejs/i, "donejs"], + [/mern/i, "mern"], + [/mean/i, "mean"], + [/mocha/i, "mocha"], + [/jasmine/i, "jasmine"], + [/enzyme/i, "enzyme"], + [/jest/i, "jest"], + [/cucumber/i, "cucumberjs"], + [/^ava$/i, "ava"], + [/^java$/i, "java"], + [/tape/i, "tape"], + [/karma/i, "karma"], + [/plain( |-|\.)css/i, "plaincss"], + [/css( |-|_|\.)?modules/i, "cssmodules"], + [/css( |-|_|\.)?next/i, "cssnext"], + [/pure( |-|_|\.)?css/i, "purecss"], + [/post( |-|_|\.)?css/i, "postcss"], + [/css( |-|_)?in( |-|_)?js/i, "css_in_js"], + [/s(a|c)ss/i, "sass"], + [/^less$/i, "less"], + [/stylus/i, "stylus"], + [/aphrodite/i, "aphrodite"], + [/webpack/i, "webpack"], + [/grunt/i, "grunt"], + [/gulp/i, "gulp"], + [/browserify/i, "browserify"], + [/bower/i, "bower"], + [/native( |-|_)?apps/i, "nativeapps"], + [/react( |-|_)?native/i, "reactnative"], + [/phonegap\/cordova/i, "cordova"], + [/cordova/i, "cordova"], + [/phonegap/i, "phonegap"], + [/native( |-|_)?script/i, "nativescript"], + [/express/i, "express"], + [/koa/i, "koa"], + [/hapi/i, "hapi"], + [/relay/i, "relay"], + [/sails/i, "sails"], + [/loopback/i, "loopback"], + [/keystone/i, "keystone"], + [/electron/i, "electron"], + [/ionic/i, "ionic"], + [/bootstrap/i, "bootstrap"], + [/foundation/i, "foundation"], + [/npm/i, "npm"], + [/rollup/i, "rollup"], + [/next(\.| |-|_)?js/i, "nextjs"], + [/storybook/i, "storybook"], + [/cycle( |-|_|\.)js/i, "cyclejs"], + [/knockout/i, "knockout"], + [/jquery/i, "jquery"], + [/mithril/i, "mithril"], + [/inferno/i, "inferno"], + [/riot/i, "riotjs"], + [/^ext\.?(js)?$/i, "extjs"], + [/svelte/i, "svelte"], + [/^om$/i, "om"], + [/choo/i, "choo"], + [/hyperapp/i, "hyperapp"], + [/dojo/i, "dojo"], + [/alkali/i, "alkali"], + [/marionette/i, "marionette"], + [/kendo/i, "kendo"], + [/marko/i, "marko"], + [/reagent/i, "reagent"], + [/haxe/i, "haxe"], + [/ampersand/i, "ampersand"], + [/rxjs/i, "rxjs"], + [/canjs/i, "canjs"], + [/prototype/i, "prototype"], + [/mootools/i, "mootools"], + [/glimmer/i, "glimmerjs"], + [/durandal/i, "durandal"], + [/moon/i, "moon"], + [/batman/i, "batman"], + [/flight/i, "flight"], + [/scala/i, "scala"], + [/webix/i, "webix"], + [/enyo/i, "enyo"], + [/quasar/i, "quasar"], + [/d3/i, "d3"], + [/three/i, "threejs"], + [/cerebral/i, "cerebral"], + [/underscore/i, "underscore"], + [/vuex/i, "vuex"], + [/flux/i, "flux"], + [/pouch/i, "pouchdb"], + [/ngrx/i, "ngrx"], + [/graph\.?cool/i, "graphcool"], + [/parse/i, "parse"], + [/realm/i, "realm"], + [/gun/i, "gunjs"], + [/couchbase/i, "couchbase"], + [/datomic/i, "datomic"], + [/deployd/i, "deployd"], + [/hood\.?ie/i, "hoodie"], + [/kinvey/i, "kinvey"], + [/kinto/i, "kinto"], + [/mongo/i, "mongodb"], + [/contentful/i, "contentful"], + [/mysql/i, "mysql"], + [/redis/i, "redis"], + [/rethink/i, "rethinkdb"], + [/^node$|node\.?js|node js/i, "nodejs"], + [/asp\. ?net|\.net|dotnet/i, "dotnet"], + [/c#/i, "csharp"], + [/django/i, "django"], + [/ror|rails|ruby on rails/i, "rails"], + [/phpstorm/i, "phpstorm"], + [/(^|\s)php($|\s)/i, "php"], + [/adonis/i, "adonis"], + [/python/i, "python"], + [/^go$|golang/i, "golang"], + [/laravel/i, "laravel"], + [/elixir/i, "elixir"], + [/couchdb/i, "couchdb"], + [/trails/i, "trails"], + [/kraken/i, "kraken"], + [/serverless/i, "serverless"], + [/socket\.?io/i, "socketio"], + [/micro/i, "micro"], + [/haskell/i, "haskell"], + [/qunit/i, "qunit"], + [/chai/i, "chai"], + [/protractor/i, "protractor"], + [/selenium/i, "selenium"], + [/nightwatch/i, "nightwatch"], + [/^tap$/i, "node_tap"], + [/sinon/i, "sinonjs"], + [/^intern$/i, "intern"], + [/^lab$/i, "lab"], + [/testcafe/i, "testcafe"], + [/junit/i, "junit"], + [/cypress/i, "cypress"], + [/phantom/i, "phantomjs"], + [/^atomic/i, "atomic_design"], + [/bem/i, "bem"], + [/bulma/i, "bulma"], + [/semantic/i, "semanticui"], + [/tach\.+on/i, "tachyons"], + [/material design/i, "material_design"], + [/skeleton/i, "skeleton"], + [/material(\.| |-|_)?ui/i, "material_ui"], + [/style(d)?(\.| |-|_)?co(m|n)ponent(s)?/i, "styled_components"], + [/bourbon/i, "bourbon"], + [/mill?igram/i, "milligram"], + [/uikit/i, "uikit"], + [/flexbox/i, "flexbox"], + [/topocoat/i, "topocoat"], + [/glamor/i, "glamor"], + [/radium/i, "radium"], + [/styletron/i, "styletron"], + [/jss/i, "jss"], + [/mdl/i, "mdl"], + [/vuetify/i, "vuetify"], + [/materiali(z|s)e/i, "materialize"], + [/brunch/i, "brunch"], + [/make/i, "make"], + [/broccoli|brocolli/i, "broccoli"], + [/fusebox/i, "fusebox"], + [/systemjs/i, "systemjs"], + [/gradle/i, "gradle"], + [/stealjs/i, "stealjs"], + [/babel/i, "babel"], + [/weex/i, "weex"], + [/xamarin/i, "xamarin"], + [/pwa/i, "pwa"], + [/progressive( |-|_)?web( |-|_)?app/i, "pwa"], + [/^nw(js)$/i, "nwjs"], + [/expo/i, "expo"], + [/flutter/i, "flutter"], + [/(appcelerator|titanium)/i, "appcelerator"], + [/fuse/i, "fuse"], + [/^cef$/i, "cef"], + [/chromium embedded framework/i, "cef"], + [/swift/i, "swift"], + [/yarn/i, "yarn"], + [/nuget/i, "nuget"], + [/composer/i, "composer"], + [/jspm/i, "jspm"], + [/pnpm/i, "pnpm"], + [/homebrew|brew/i, "homebrew"], + [/leiningen/i, "leiningen"], + [/maven/i, "maven"], + [/immutable/i, "immutable"], + [/lodash/i, "lodash"], + [/moment/i, "moment"], + [/ramda/i, "ramda"], + [/axios/i, "axios"], + [/recompose/i, "recompose"], + [/zepto/i, "zepto"], + [/async/i, "async"], + [/atom/i, "atom"], + [/webstorm/i, "webstorm"], + [/intellij/i, "intellij"], + [/sublime/i, "sublime-text"], + [/vim/i, "vim"], + [/emacs/i, "emacs"], + [/brackets/i, "brackets"], + [/(visual studio|vs code|vscode|visualstudio)/i, "visual_studio"], + [/notepad/i, "notepad++"], + [/netbeans/i, "netbeans"], + [/coda/i, "coda"], + [/prettier/i, "prettier"], + [/eslint/i, "eslint"], + [/tslint/i, "tslint"], + [/stylelint/i, "stylelint"], + [/jshint/i, "jshint"], + [/standardjs/i, "standardjs"], + [/jscs/i, "jscs"], + [/jslint/i, "jslint"], + [/^xo$/i, "xo"], + [/date ?fns/i, "date_fns"], + [/date(-|_)?fns/i, "date_fns"], + [/objective(-|_)?c/i, "objective_c"], + [/objective ?c/i, "objective_c"], + [/parcel/i, "parcel"], + [/rust/i, "rust"], + [/service(-|_)?workers/i, "service_workers"], + [/ocaml/i, "ocaml"], + [/nano/i, "nano"], + [/kotlin/i, "kotlin"], + [/^r$/i, "r"], + [/pycharm/i, "pycharm"], + [/erlang/i, "erlang"], + [/bash/i, "bash"], + [/delphi/i, "delphi"], + [/perl/i, "perl"], + [/lisp/i, "lisp"], + [/purescript/i, "purescript"], + [/websockets/i, "websockets"], + [/luxon/i, "luxon"], + [/fbelt/i, "fbelt"], + [/bbedit/i, "bbedit"], + [/dart/i, "dart"], + [/lua/i, "lua"], + + [/ruby/i, "ruby"], + [/LitElement/i, "litelement"], + + [/nest/i, "nest"], + [/hapi/i, "hapi"], + [/gridsome/i, "gridsome"], + [/sapper/i, "sapper"], + [/vulcan/i, "vulcan"], + [/prisma/i, "prisma"], + [/flask/i, "flask"], + [/inertia/i, "inertia"], + [/egg/i, "egg"], + [/strapi/i, "strapi"], + [/socket\.io/i, "socketio"], + [/socketio/i, "socketio"], + [/fastify/i, "fastify"], + [/restify/i, "restify"], + [/symfony/i, "symfony"], + + [/stimulus/i, "stimulus"], + [/stencil/i, "stencil"], + [/re\-frame/i, "reframe"], + [/reframe/i, "reframe"], + [/lit-html/i, "litelement"], + [/lit-element/i, "litelement"], + [/hyperhtml/i, "hyperhtml"], + [/cycle/i, "cycle"], + [/sencha/i, "sencha"], + [/ractive/i, "ractive"], + [/nuxt/i, "nuxt"], + [/gatsby/i, "gatsby"], + [/blazor/i, "blazor"], + [/angularjs/i, "angularjs"], + + [/xstate/i, "xstate"], + [/urql/i, "urql"], + [/unistore/i, "unistore"], + [/unistroe/i, "unistore"], + [/storeon/i, "storeon"], + [/stator/i, "stator"], + [/orbit/i, "orbit"], + [/immer/i, "immer"], + [/hasura/i, "hasura"], + [/easy\-peasy/i, "easypeasy"], + [/easypeasy/i, "easypeasy"], + [/unstated/i, "unstated"], + [/swr/i, "swr"], + [/rematch/i, "rematch"], + [/context/i, "react_context"], + [/hooks/i, "react_hooks"], + [/overmind/i, "overmind"], + [/ngxs/i, "ngxs"], + [/effector/i, "effector"], + [/akita/i, "akita"], + + [/webdriver/i, "webdriver"], + [/wallaby/i, "wallaby"], + [/vue\-test\-utils/i, "vue_test_utils"], + [/testing\-library/i, "react_testing_library"], + [/testing ?library/i, "react_testing_library"], + [/styleguidist/i, "styleguidist"], + [/stryker/i, "stryker"], + [/spectator/i, "spectator"], + [/stryker/i, "stryker"], + [/percy/i, "percy"], + [/detox/i, "detox"], + [/codecept/i, "codecept"], + + [/capacitor/i, "capacitor"], + [/vue native/i, "vue_native"], + [/revery/i, "revery"], + [/nodegui/i, "nodegui"], + [/node gui/i, "nodegui"], + [/framework7/i, "framework7"], + + [/vivaldi/i, "vivaldi"], + [/qute/i, "qutebrowser"], + [/opera/i, "opera"], + [/brave/i, "brave"], + [/sizzy/i, "sizzy"], + [/internet explorer 11/i, "internet-explorer-11"], + [/ie11/i, "internet-explorer-11"], + [/ie 11/i, "internet-explorer-11"], + [/firefox developer edition/i, "firefox_developer_edition"], + [/edge chromium/i, "edge"], + [/edge \(chromium\)/i, "edge"], + [/chromium/i, "chromium"], + [/edge/i, "edge"], + [/chrome/i, "chrome"], + [/safari/i, "safari"], + [/firefox/i, "firefox"], + + [/tsc/i, "tsc"], + [/shadow/i, "shadow_cljs"], + [/pika/i, "pika"], + [/lerna/i, "lerna"], + [/create-react-app/i, "create_react_app"], + [/cra/i, "create_react_app"], + [/angular-cli/i, "angular_cli"], + [/angular cli/i, "angular_cli"], + [/vue-cli/i, "vue_cli"], + [/vue cli/i, "vue_cli"], + [/metro/i, "metro"], + [/bazel/i, "bazel"], + [/codekit/i, "codekit"], + + [/voca/i, "voca"], + [/uuid/i, "uuid"], + [/spacetime/i, "spacetime"], + [/reselect/i, "reselect"], + [/numeral/i, "numeral"], + [/math/i, "mathjs"], + [/joda/i, "js_joda"], + [/i18next/i, "i18next"], + [/gsap/i, "gsap"], + [/fp-ts/i, "fp_ts"], + [/formik/i, "formik"], + [/dotenv/i, "dotenv"], + [/day\.js/i, "dayjs"], + [/dayjs/i, "dayjs"], + [/classnames/i, "classnames"], + [/bluebird/i, "bluebird"], + [/sanctuary/i, "sanctuary"], + [/rambda/i, "ramda"], + [/nodemon/i, "nodemon"], + [/greensock/i, "gsap"], + [/emotion/i, "emotion"], + + [/sql/i, "sql"], + [/shell/i, "shell"], + [/solidity/i, "solidity"], + [/matlab/i, "matlab"], + [/html/i, "html"], + [/css/i, "css"], + [/groovy/i, "groovy"], + [/f\#/i, "fsharp"], + [/crystal/i, "crystal"], + [/coldfusion/i, "coldfusion"], + [/cfml/i, "coldfusion"], + [/apex/i, "apex"], +]; + +export default [ + ...sourceNormalizationRules, + ...toolNormalizationRules, + ...otherFeaturesNormalizationRules, +]; diff --git a/dataops/package.json b/dataops/package.json new file mode 100644 index 000000000..ff53eb05e --- /dev/null +++ b/dataops/package.json @@ -0,0 +1,13 @@ +{ + "dependencies": { + "lodash": "^4.17.21", + "mongo": "^0.1.0", + "mongodb": "^4.9.0" + }, + "devDependencies": { + "@types/lodash": "^4.14.184", + "@types/node": "^18.7.14", + "i": "^0.3.7", + "typescript": "^4.8.2" + } +} diff --git a/dataops/pnpm-lock.yaml b/dataops/pnpm-lock.yaml new file mode 100644 index 000000000..5e583799f --- /dev/null +++ b/dataops/pnpm-lock.yaml @@ -0,0 +1,172 @@ +lockfileVersion: 5.4 + +specifiers: + '@types/lodash': ^4.14.184 + '@types/node': ^18.7.14 + i: ^0.3.7 + lodash: ^4.17.21 + mongo: ^0.1.0 + mongodb: ^4.9.0 + typescript: ^4.8.2 + +dependencies: + lodash: 4.17.21 + mongo: 0.1.0 + mongodb: 4.9.0 + +devDependencies: + '@types/lodash': 4.14.184 + '@types/node': 18.7.14 + i: 0.3.7 + typescript: 4.8.2 + +packages: + + /@types/lodash/4.14.184: + resolution: {integrity: sha512-RoZphVtHbxPZizt4IcILciSWiC6dcn+eZ8oX9IWEYfDMcocdd42f7NPI6fQj+6zI8y4E0L7gu2pcZKLGTRaV9Q==} + dev: true + + /@types/node/18.7.14: + resolution: {integrity: sha512-6bbDaETVi8oyIARulOE9qF1/Qdi/23z6emrUh0fNJRUmjznqrixD4MpGDdgOFk5Xb0m2H6Xu42JGdvAxaJR/wA==} + + /@types/webidl-conversions/7.0.0: + resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==} + dev: false + + /@types/whatwg-url/8.2.2: + resolution: {integrity: sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==} + dependencies: + '@types/node': 18.7.14 + '@types/webidl-conversions': 7.0.0 + dev: false + + /base64-js/1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + dev: false + + /bson/4.7.0: + resolution: {integrity: sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==} + engines: {node: '>=6.9.0'} + dependencies: + buffer: 5.7.1 + dev: false + + /buffer/5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + dev: false + + /denque/2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + dev: false + + /i/0.3.7: + resolution: {integrity: sha512-FYz4wlXgkQwIPqhzC5TdNMLSE5+GS1IIDJZY/1ZiEPCT2S3COUVZeT5OW4BmW4r5LHLQuOosSwsvnroG9GR59Q==} + engines: {node: '>=0.4'} + dev: true + + /ieee754/1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + dev: false + + /ip/2.0.0: + resolution: {integrity: sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==} + dev: false + + /lodash/4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + dev: false + + /memory-pager/1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + dev: false + optional: true + + /mongo/0.1.0: + resolution: {integrity: sha512-2MPq+GCNKhah0V/g/HIQI/S1h6Ycd87KPuXAITkeXWT6wncvABFxOaXdzCKlRvLSQRUkDimllrRrhoHUTD8usg==} + engines: {node: '>= 0.4.0'} + dependencies: + mongodb: 4.9.0 + dev: false + + /mongodb-connection-string-url/2.5.3: + resolution: {integrity: sha512-f+/WsED+xF4B74l3k9V/XkTVj5/fxFH2o5ToKXd8Iyi5UhM+sO9u0Ape17Mvl/GkZaFtM0HQnzAG5OTmhKw+tQ==} + dependencies: + '@types/whatwg-url': 8.2.2 + whatwg-url: 11.0.0 + dev: false + + /mongodb/4.9.0: + resolution: {integrity: sha512-tJJEFJz7OQTQPZeVHZJIeSOjMRqc5eSyXTt86vSQENEErpkiG7279tM/GT5AVZ7TgXNh9HQxoa2ZkbrANz5GQw==} + engines: {node: '>=12.9.0'} + dependencies: + bson: 4.7.0 + denque: 2.1.0 + mongodb-connection-string-url: 2.5.3 + socks: 2.7.0 + optionalDependencies: + saslprep: 1.0.3 + dev: false + + /punycode/2.1.1: + resolution: {integrity: sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==} + engines: {node: '>=6'} + dev: false + + /saslprep/1.0.3: + resolution: {integrity: sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==} + engines: {node: '>=6'} + requiresBuild: true + dependencies: + sparse-bitfield: 3.0.3 + dev: false + optional: true + + /smart-buffer/4.2.0: + resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} + engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} + dev: false + + /socks/2.7.0: + resolution: {integrity: sha512-scnOe9y4VuiNUULJN72GrM26BNOjVsfPXI+j+98PkyEfsIXroa5ofyjT+FzGvn/xHs73U2JtoBYAVx9Hl4quSA==} + engines: {node: '>= 10.13.0', npm: '>= 3.0.0'} + dependencies: + ip: 2.0.0 + smart-buffer: 4.2.0 + dev: false + + /sparse-bitfield/3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + dependencies: + memory-pager: 1.5.0 + dev: false + optional: true + + /tr46/3.0.0: + resolution: {integrity: sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==} + engines: {node: '>=12'} + dependencies: + punycode: 2.1.1 + dev: false + + /typescript/4.8.2: + resolution: {integrity: sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==} + engines: {node: '>=4.2.0'} + hasBin: true + dev: true + + /webidl-conversions/7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + dev: false + + /whatwg-url/11.0.0: + resolution: {integrity: sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==} + engines: {node: '>=12'} + dependencies: + tr46: 3.0.0 + webidl-conversions: 7.0.0 + dev: false diff --git a/surveyform/src/server/scripts.ts b/dataops/scripts.ts similarity index 74% rename from surveyform/src/server/scripts.ts rename to dataops/scripts.ts index c2de4b6e0..47306af34 100644 --- a/surveyform/src/server/scripts.ts +++ b/dataops/scripts.ts @@ -1,18 +1,20 @@ -import type { Model } from "mongoose"; -import { normalizeResponse } from "../admin/server/normalization/normalize"; -import { fetchEntities } from "~/modules/entities/server/graphql"; +// import { normalizeResponse } from "../admin/server/normalization/normalize"; +// import { fetchEntities } from "~/modules/entities/server/graphql"; +import { fetchEntities } from "./fetchEntities"; + // import Users from 'meteor/vulcan:users'; //import { // js2019FieldMigrations, // normalizeJS2019Value, // otherValueNormalisations, //} from "./migrations"; -import { ResponseMongooseModel } from "~/modules/responses/model.server"; // import { NormalizedResponseMongooseModel } from "~/modules/normalized_responses/model.server"; -import { createEmailHash } from "~/account/email/api/encryptEmail"; -import { UserMongooseModel } from "~/core/models/user.server"; -import { connectToAppDb } from "~/lib/server/mongoose/connection"; -import { logToFile } from "~/lib/server/debug"; +//import { createEmailHash } from "~/account/email/api/encryptEmail"; +// import { logToFile } from "~/lib/server/debug"; +import { createEmailHash } from "../shared/server/createEmailHash"; +import { logToFile } from "../shared/server/debug"; +import type { Collection } from "mongodb"; +import { getResponseCollection } from "./mongo/models"; /* @@ -20,20 +22,21 @@ Migrations */ export const renameFieldMigration = async ( - collection: Model, - field1, - field2 + collection: Collection, + field1: string, + field2: string, ) => { - await connectToAppDb(); + // Make sure you are connected before this call + // await connectToAppDb(); const result = await collection.updateMany( { [field1]: { $exists: true } }, - { $rename: { [field1]: field2 } } + { $rename: { [field1]: field2 } }, ); // eslint-disable-next-line no-console console.log( - `// ${field1} -> ${field2} migration done, renamed ${result.modifiedCount} fields ` + `// ${field1} -> ${field2} migration done, renamed ${result.modifiedCount} fields `, ); }; @@ -42,8 +45,8 @@ export const renameFieldMigration = async ( Renormalize a survey's results */ -const renormalizeSurvey = async (surveySlug) => { - await connectToAppDb(); +const renormalizeSurvey = async (surveySlug: string) => { + const ResponseCollection = await getResponseCollection(); const fileName = `${surveySlug}_normalization`; const limit = 99999; @@ -65,23 +68,23 @@ const renormalizeSurvey = async (surveySlug) => { const startAt = new Date(); let progress = 0; - const responsesCursor = ResponseMongooseModel.find(selector, null, { limit }); + const responsesCursor = ResponseCollection.find(selector, { limit }); const count = await responsesCursor.clone().count(); const tickInterval = Math.round(count / 200); await logToFile( `${fileName}.txt`, "id, fieldName, value, matchTags, id, pattern, rules, match \n", - { mode: "overwrite" } + { mode: "overwrite" }, ); await logToFile(`${fileName}.txt`, "", { mode: "overwrite" }); await logToFile("normalization_errors.txt", "", { mode: "overwrite" }); console.log( - `// Renormalizing survey ${surveySlug}… Found ${count} responses to renormalize. (${startAt})` + `// Renormalizing survey ${surveySlug}… Found ${count} responses to renormalize. (${startAt})`, ); - const responses = await responsesCursor.clone().exec(); + const responses = await responsesCursor.clone().toArray(); for (const response of responses) { try { // console.log(progress, progress % tickInterval, response._id); @@ -117,10 +120,10 @@ const renormalizeSurvey = async (surveySlug) => { const endAt = new Date(); const duration = Math.ceil( - (endAt.valueOf() - startAt.valueOf()) / (1000 * 60) + (endAt.valueOf() - startAt.valueOf()) / (1000 * 60), ); console.log( - `-> Done renormalizing ${count} responses in survey ${surveySlug}. (${endAt}) - ${duration} min` + `-> Done renormalizing ${count} responses in survey ${surveySlug}. (${endAt}) - ${duration} min`, ); }; @@ -130,9 +133,9 @@ Log all "currently missing features from CSS" answers to file */ export const logField = async ( - collection: Model, - fieldName, - surveySlug + collection: Collection, + fieldName: string, + surveySlug: string, ) => { const selector: any = { [fieldName]: { @@ -150,27 +153,29 @@ export const logField = async ( }; export async function migrateUserEmails() { + const db = await connectToAppDb(); + const UserCollection = db.collection("users"); console.log("// migrateUserEmails"); // get all users that have a plain-text email stored - const usersToMigrate = await UserMongooseModel.find({ + const usersToMigrate = await UserCollection.find({ email: { $exists: true }, legacyEmailHash: { $exists: false }, - }); + }).toArray(); console.log(`// Found ${usersToMigrate.length} users to migrate…`); for (const user of usersToMigrate) { const { _id, email, emailHash: legacyEmailHash } = user; const newEmailHash = createEmailHash(email); const set = { legacyEmailHash, emailHash: newEmailHash }; console.log( - `// Updating user ${_id}, email: ${email}, old hash: ${legacyEmailHash}, new hash: ${newEmailHash}` + `// Updating user ${_id}, email: ${email}, old hash: ${legacyEmailHash}, new hash: ${newEmailHash}`, ); const unset = {}; // for now keep emails for safety, in the future delete them // const unset = { email: 1, emails: 1} - const update = await UserMongooseModel.updateOne( + const update = await UserCollection.updateOne( { _id }, - { $set: set, $unset: unset } + { $set: set, $unset: unset }, ); } } @@ -204,8 +209,15 @@ export const renameGraphQL2022Fields = async () => { // await renameFieldMigration(ResponseMongooseModel, 'graphql2022__usage_others__strong_points', 'graphql2022__usage_others__graphql_strong_points'); // await renameFieldMigration(ResponseMongooseModel, 'graphql2022__usage_others__pain_points', 'graphql2022__usage_others__graphql_pain_points'); - await renameFieldMigration(ResponseMongooseModel, 'graphql2022__usage__graphql_experience', 'graphql2022__usage__graphql_experience__choices'); - await renameFieldMigration(ResponseMongooseModel, 'graphql2022__usage__code_generation_type', 'graphql2022__usage__code_generation_type__choices'); - - + const ResponseCollection = await getResponseCollection(); + await renameFieldMigration( + ResponseCollection, + "graphql2022__usage__graphql_experience", + "graphql2022__usage__graphql_experience__choices", + ); + await renameFieldMigration( + ResponseCollection, + "graphql2022__usage__code_generation_type", + "graphql2022__usage__code_generation_type__choices", + ); }; diff --git a/dataops/tsconfig.json b/dataops/tsconfig.json new file mode 100644 index 000000000..75dcaeac2 --- /dev/null +++ b/dataops/tsconfig.json @@ -0,0 +1,103 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ + // "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + // "outDir": "./", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +} diff --git a/dataops/yarn.lock b/dataops/yarn.lock new file mode 100644 index 000000000..0174f5001 --- /dev/null +++ b/dataops/yarn.lock @@ -0,0 +1,145 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@types/node@*", "@types/node@^18.7.14": + version "18.7.14" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.7.14.tgz#0fe081752a3333392d00586d815485a17c2cf3c9" + integrity sha512-6bbDaETVi8oyIARulOE9qF1/Qdi/23z6emrUh0fNJRUmjznqrixD4MpGDdgOFk5Xb0m2H6Xu42JGdvAxaJR/wA== + +"@types/webidl-conversions@*": + version "7.0.0" + resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz#2b8e60e33906459219aa587e9d1a612ae994cfe7" + integrity sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog== + +"@types/whatwg-url@^8.2.1": + version "8.2.2" + resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-8.2.2.tgz#749d5b3873e845897ada99be4448041d4cc39e63" + integrity sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA== + dependencies: + "@types/node" "*" + "@types/webidl-conversions" "*" + +base64-js@^1.3.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + +bson@^4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/bson/-/bson-4.7.0.tgz#7874a60091ffc7a45c5dd2973b5cad7cded9718a" + integrity sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA== + dependencies: + buffer "^5.6.0" + +buffer@^5.6.0: + version "5.7.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" + integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.1.13" + +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + +ieee754@^1.1.13: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== + +ip@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.0.tgz#4cf4ab182fee2314c75ede1276f8c80b479936da" + integrity sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ== + +memory-pager@^1.0.2: + version "1.5.0" + resolved "https://registry.yarnpkg.com/memory-pager/-/memory-pager-1.5.0.tgz#d8751655d22d384682741c972f2c3d6dfa3e66b5" + integrity sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg== + +mongo@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/mongo/-/mongo-0.1.0.tgz#c8af0f8df98d4894b772b37342987c3becf3718c" + integrity sha512-2MPq+GCNKhah0V/g/HIQI/S1h6Ycd87KPuXAITkeXWT6wncvABFxOaXdzCKlRvLSQRUkDimllrRrhoHUTD8usg== + dependencies: + mongodb "*" + +mongodb-connection-string-url@^2.5.3: + version "2.5.3" + resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-2.5.3.tgz#c0c572b71570e58be2bd52b33dffd1330cfb6990" + integrity sha512-f+/WsED+xF4B74l3k9V/XkTVj5/fxFH2o5ToKXd8Iyi5UhM+sO9u0Ape17Mvl/GkZaFtM0HQnzAG5OTmhKw+tQ== + dependencies: + "@types/whatwg-url" "^8.2.1" + whatwg-url "^11.0.0" + +mongodb@*, mongodb@^4.9.0: + version "4.9.0" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-4.9.0.tgz#58618439b721f2d6f7d38bb10a4612e29d7f1c8a" + integrity sha512-tJJEFJz7OQTQPZeVHZJIeSOjMRqc5eSyXTt86vSQENEErpkiG7279tM/GT5AVZ7TgXNh9HQxoa2ZkbrANz5GQw== + dependencies: + bson "^4.7.0" + denque "^2.1.0" + mongodb-connection-string-url "^2.5.3" + socks "^2.7.0" + optionalDependencies: + saslprep "^1.0.3" + +punycode@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + +saslprep@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/saslprep/-/saslprep-1.0.3.tgz#4c02f946b56cf54297e347ba1093e7acac4cf226" + integrity sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag== + dependencies: + sparse-bitfield "^3.0.3" + +smart-buffer@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" + integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== + +socks@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.0.tgz#f9225acdb841e874dca25f870e9130990f3913d0" + integrity sha512-scnOe9y4VuiNUULJN72GrM26BNOjVsfPXI+j+98PkyEfsIXroa5ofyjT+FzGvn/xHs73U2JtoBYAVx9Hl4quSA== + dependencies: + ip "^2.0.0" + smart-buffer "^4.2.0" + +sparse-bitfield@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz#ff4ae6e68656056ba4b3e792ab3334d38273ca11" + integrity sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ== + dependencies: + memory-pager "^1.0.2" + +tr46@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-3.0.0.tgz#555c4e297a950617e8eeddef633c87d4d9d6cbf9" + integrity sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA== + dependencies: + punycode "^2.1.1" + +typescript@^4.8.2: + version "4.8.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.8.2.tgz#e3b33d5ccfb5914e4eeab6699cf208adee3fd790" + integrity sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw== + +webidl-conversions@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" + integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== + +whatwg-url@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" + integrity sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ== + dependencies: + tr46 "^3.0.0" + webidl-conversions "^7.0.0" diff --git a/shared/package.json b/shared/package.json new file mode 100644 index 000000000..8b4adaf8f --- /dev/null +++ b/shared/package.json @@ -0,0 +1,6 @@ +{ + "devDependencies": { + "@types/node": "^18.7.14", + "typescript": "^4.8.2" + } +} diff --git a/shared/pnpm-lock.yaml b/shared/pnpm-lock.yaml new file mode 100644 index 000000000..c80c48878 --- /dev/null +++ b/shared/pnpm-lock.yaml @@ -0,0 +1,21 @@ +lockfileVersion: 5.4 + +specifiers: + '@types/node': ^18.7.14 + typescript: ^4.8.2 + +devDependencies: + '@types/node': 18.7.14 + typescript: 4.8.2 + +packages: + + /@types/node/18.7.14: + resolution: {integrity: sha512-6bbDaETVi8oyIARulOE9qF1/Qdi/23z6emrUh0fNJRUmjznqrixD4MpGDdgOFk5Xb0m2H6Xu42JGdvAxaJR/wA==} + dev: true + + /typescript/4.8.2: + resolution: {integrity: sha512-C0I1UsrrDHo2fYI5oaCGbSejwX4ch+9Y5jTQELvovfmFkK3HHSZJB8MSJcWLmCUBzQBchCrZ9rMRV6GuNrvGtw==} + engines: {node: '>=4.2.0'} + hasBin: true + dev: true diff --git a/shared/server/createEmailHash.ts b/shared/server/createEmailHash.ts new file mode 100644 index 000000000..aa3f4f693 --- /dev/null +++ b/shared/server/createEmailHash.ts @@ -0,0 +1,20 @@ +import crypto from "crypto"; + +/** + * Creating Hash from Emails, not reversible + */ +export const createEmailHash = (email: string) => { + const hashSaltStr = process.env.HASH_SALT || + //getSetting('hashSalt') || + process.env.ENCRYPTION_KEY; //|| + //getSetting('encriptionKey') + if (!hashSaltStr) { + throw new Error(`HASH_SALT/ENCRYPTION_KEY environment variable not set`); + } + + const hashSalt = Buffer.from(hashSaltStr); + const hash = crypto.createHash("sha512-256WithRSAEncryption"); + hash.update(hashSalt); + hash.update(email); + return hash.digest("hex"); +}; diff --git a/shared/server/debug.ts b/shared/server/debug.ts new file mode 100644 index 000000000..9e33d0385 --- /dev/null +++ b/shared/server/debug.ts @@ -0,0 +1,66 @@ +//import path from path +import fs from "fs"; +import path from "path"; +const logsDirectory = ".logs"; + +/** + * NOTE: not working on Windows yet + * @param fileName + * @param object + * @param options + */ +export const logToFile = async ( + fileName: string, + object: string | Object, + options: { mode?: "append" | "overwrite"; timestamp?: boolean } = {}, +) => { + const { mode = "append", timestamp = false } = options; + // the server path is of type "/Users/foo/bar/appName/.meteor/local/build/programs/server" + // we remove the last five segments to get the app directory + // eslint-disable-next-line no-undef + const serverDir = process.env.VERCEL + ? "/tmp" + : path.resolve(__dirname, "../../../"); + const filePath = serverDir.split(path.sep).slice(1, -5).join(path.sep); //__meteor_bootstrap__.serverDir + const logsDirPath = `/${filePath}/${logsDirectory}`; + if (!fs.existsSync(logsDirPath)) { + fs.mkdirSync(logsDirPath, { recursive: true }); + } + const fullPath = `${logsDirPath}/${fileName}`; + const contents = typeof object === "string" + ? object + : JSON.stringify(object, null, 2); + const now = new Date(); + const text = timestamp ? now.toString() + "\n---\n" + contents : contents; + if (mode === "append") { + const stream = fs.createWriteStream(fullPath, { flags: "a" }); + stream.write(text + "\n"); + stream.end(); + } else { + fs.readFile(fullPath, (error, data) => { + let shouldWrite = false; + if (error && error.code === "ENOENT") { + // the file just does not exist, ok to write + shouldWrite = true; + } else if (error) { + // maybe EACCESS or something wrong with the disk + throw error; + } else { + const fileContent = data.toString(); + if (fileContent !== text) { + shouldWrite = true; + } + } + + if (shouldWrite) { + fs.writeFile(fullPath, text, (error) => { + // throws an error, you could also catch it here + if (error) throw error; + + // eslint-disable-next-line no-console + console.log(`New graphql schema saved to ${fullPath}`); + }); + } + }); + } +}; diff --git a/shared/tsconfig.json b/shared/tsconfig.json new file mode 100644 index 000000000..75dcaeac2 --- /dev/null +++ b/shared/tsconfig.json @@ -0,0 +1,103 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ + // "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + // "outDir": "./", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +} diff --git a/surveyform/src/modules/entities/server/graphql.ts b/surveyform/src/modules/entities/server/graphql.ts index d0d383d9b..5f6b5e5b9 100644 --- a/surveyform/src/modules/entities/server/graphql.ts +++ b/surveyform/src/modules/entities/server/graphql.ts @@ -69,7 +69,7 @@ interface Entity { } /** - * Fetch raw entities from the trasnlation API + * Fetch raw entities from the translation API * @returns */ export const fetchEntities = async () => { @@ -94,7 +94,6 @@ export const fetchEntities = async () => { const ENTITIES_PROMISE_TTL_SECONDS = 10 * 60; /** - * * @param root * @param args * @returns @@ -127,7 +126,7 @@ export const getOrFetchEntities = async ({ let entities = await cachedPromise( promisesNodeCache, entitiesPromiseCacheKey, - ENTITIES_PROMISE_TTL_SECONDS + ENTITIES_PROMISE_TTL_SECONDS, )(async () => await fetchEntities()); // const { tags, name, id, ids } = args; diff --git a/surveyform/src/modules/responses/model.server.ts b/surveyform/src/modules/responses/model.server.ts index 851a0cc6b..97fc35a6b 100644 --- a/surveyform/src/modules/responses/model.server.ts +++ b/surveyform/src/modules/responses/model.server.ts @@ -7,7 +7,6 @@ import { modelDef as modelDefCommon } from "./model"; import { schema as schemaServer } from "./schema.server"; import { createMongooseConnector } from "@vulcanjs/mongo"; import { subscribe } from "~/server/email/email_octopus"; -import { createEmailHash } from "~/account/email/api/encryptEmail"; import { surveyFromResponse } from "~/modules/responses/helpers"; // import { updateElasticSearchOnCreate, updateElasticSearchOnUpdate } from '../elasticsearch/index'; @@ -92,13 +91,13 @@ export const ResponseConnector = createMongooseConnector( Response, { mongooseSchema: new mongoose.Schema({ _id: String }, { strict: false }), - } + }, ); Response.crud.connector = ResponseConnector; // Using Mongoose (advised) -export const ResponseMongooseModel = - ResponseConnector.getRawCollection() as mongoose.Model; +export const ResponseMongooseModel = ResponseConnector + .getRawCollection() as mongoose.Model; /** * For direct Mongo access (not advised, used only for aggregations) @@ -108,7 +107,7 @@ export const ResponseMongooseModel = export const ResponseMongoCollection = () => { if (!mongoose.connection.db) { throw new Error( - "Trying to access Response mongo collection before Mongo/Mongoose is connected." + "Trying to access Response mongo collection before Mongo/Mongoose is connected.", ); } return mongoose.connection.db.collection("responses");