1745 lines
57 KiB
TypeScript
1745 lines
57 KiB
TypeScript
import mustache from 'mustache'
|
||
import express, { response } from 'express'
|
||
import axios from 'axios'
|
||
import { SolrRecord } from './types/solrrecord'
|
||
import _ from 'lodash'
|
||
import bodyParser from 'body-parser'
|
||
import { v4 as uuidv4, v4 } from 'uuid'
|
||
import ss from 'simple-statistics'
|
||
import ip from 'ip'
|
||
import crypto from 'crypto'
|
||
import { spawn } from 'child_process'
|
||
|
||
// import fetch from 'node-fetch';
|
||
|
||
const stateMapping: Record<string, string> = {
|
||
"Alabama": "AL",
|
||
"Kentucky": "KY",
|
||
"Ohio": "OH",
|
||
"Alaska": "AK",
|
||
"Louisiana": "LA",
|
||
"Oklahoma": "OK",
|
||
"Arizona": "AZ",
|
||
"Maine": "ME",
|
||
"Oregon": "OR",
|
||
"Arkansas": "AR",
|
||
"Maryland": "MD",
|
||
"Pennsylvania": "PA",
|
||
"American Samoa": "AS",
|
||
"Massachusetts": "MA",
|
||
"Puerto Rico": "PR",
|
||
"California": "CA",
|
||
"Michigan": "MI",
|
||
"Rhode Island": "RI",
|
||
"Colorado": "CO",
|
||
"Minnesota": "MN",
|
||
"South Carolina": "SC",
|
||
"Connecticut": "CT",
|
||
"Mississippi": "MS",
|
||
"South Dakota": "SD",
|
||
"Delaware": "DE",
|
||
"Missouri": "MO",
|
||
"Tennessee": "TN",
|
||
"District of Columbia": "DC",
|
||
"Montana": "MT",
|
||
"Texas": "TX",
|
||
"Florida": "FL",
|
||
"Nebraska": "NE",
|
||
"Trust Territories": "TT",
|
||
"Georgia": "GA",
|
||
"Nevada": "NV",
|
||
"Utah": "UT",
|
||
"Guam": "GU",
|
||
"New Hampshire": "NH",
|
||
"Vermont": "VT",
|
||
"Hawaii": "HI",
|
||
"New Jersey": "NJ",
|
||
"Virginia": "VA",
|
||
"Idaho": "ID",
|
||
"New Mexico": "NM",
|
||
"Virgin Islands": "VI",
|
||
"Illinois": "IL",
|
||
"New York": "NY",
|
||
"Washington": "WA",
|
||
"Indiana": "IN",
|
||
"North Carolina": "NC",
|
||
"West Virginia": "WV",
|
||
"Iowa": "IA",
|
||
"North Dakota": "ND",
|
||
"Wisconsin": "WI",
|
||
"Kansas": "KS",
|
||
"Northern Mariana Islands": "MP",
|
||
"Wyoming": "WY"
|
||
}
|
||
|
||
const streetSuffixMapping: Record<string, string> = {
|
||
"Alley": "aly",
|
||
"Alleyway": "alyw",
|
||
"Anex": "anx",
|
||
"Annex": "anx",
|
||
"Arcade": "arc",
|
||
"Avenue": "ave",
|
||
"Bayou": "byu",
|
||
"Beach": "bch",
|
||
"Bend": "bnd",
|
||
"Bluff": "blf",
|
||
"Bottom": "btm",
|
||
"Boulevard": "blvd",
|
||
"Branch": "br",
|
||
"Bridge": "brg",
|
||
"Brook": "brk",
|
||
"Bypass": "byp",
|
||
"Camp": "cp",
|
||
"Canyon": "cyn",
|
||
"Cape": "cpe",
|
||
"Causeway": "cswy",
|
||
"Center": "ctr",
|
||
"Circle": "cir",
|
||
"Cliff": "clf",
|
||
"Club": "clb",
|
||
"Common": "cmn",
|
||
"Corner": "cor",
|
||
"Course": "crse",
|
||
"Court": "ct",
|
||
"Courts": "cts",
|
||
"Cove": "cv",
|
||
"Creek": "crk",
|
||
"Crescent": "cres",
|
||
"Crest": "crst",
|
||
"Crossing": "xing",
|
||
"Crossroad": "xrd",
|
||
"Curve": "curv",
|
||
"Dale": "dl",
|
||
"Dam": "dm",
|
||
"Drive": "dr",
|
||
"Estate": "est",
|
||
"Expressway": "expy",
|
||
"Extension": "ext",
|
||
"Falls": "fls",
|
||
"Ferry": "fry",
|
||
"Field": "fld",
|
||
"Flat": "flt",
|
||
"Ford": "frd",
|
||
"Forest": "frst",
|
||
"Forge": "frg",
|
||
"Fork": "frk",
|
||
"Fort": "ft",
|
||
"Freeway": "fwy",
|
||
"Garden": "gdn",
|
||
"Gateway": "gtwy",
|
||
"Glen": "gln",
|
||
"Green": "grn",
|
||
"Grove": "grv",
|
||
"Harbor": "hbr",
|
||
"Haven": "hvn",
|
||
"Heights": "hts",
|
||
"Highway": "hwy",
|
||
"Hill": "hl",
|
||
"Hollow": "holw",
|
||
"Inlet": "inlt",
|
||
"Island": "is",
|
||
"Junction": "jct",
|
||
"Key": "ky",
|
||
"Knoll": "knl",
|
||
"Lake": "lk",
|
||
"Landing": "lndg",
|
||
"Lane": "ln",
|
||
"Light": "lgt",
|
||
"Loaf": "lf",
|
||
"Lock": "lck",
|
||
"Locks": "lcks",
|
||
"Lodge": "ldg",
|
||
"Loop": "loop",
|
||
"Mall": "mall",
|
||
"Manor": "mnr",
|
||
"Meadow": "mdw",
|
||
"Mews": "mews",
|
||
"Mill": "ml",
|
||
"Mission": "msn",
|
||
"Motorway": "mtwy",
|
||
"Mount": "mt",
|
||
"Mountain": "mtn",
|
||
"Neck": "nck",
|
||
"Orchard": "orch",
|
||
"Overpass": "opas",
|
||
"Park": "park",
|
||
"Parkway": "pkwy",
|
||
"Pass": "pass",
|
||
"Path": "path",
|
||
"Pike": "pike",
|
||
"Pine": "pne",
|
||
"Place": "pl",
|
||
"Plain": "pln",
|
||
"Plaza": "plz",
|
||
"Point": "pt",
|
||
"Port": "prt",
|
||
"Prairie": "pr",
|
||
"Radial": "radl",
|
||
"Ramp": "ramp",
|
||
"Ranch": "rnch",
|
||
"Rapid": "rpd",
|
||
"Rest": "rst",
|
||
"Ridge": "rdg",
|
||
"River": "riv",
|
||
"Road": "rd",
|
||
"Route": "rte",
|
||
"Row": "row",
|
||
"Rue": "rue",
|
||
"Run": "run",
|
||
"Shore": "shr",
|
||
"Skyway": "skwy",
|
||
"Spring": "spg",
|
||
"Springs": "spgs",
|
||
"Spur": "spur",
|
||
"Square": "sq",
|
||
"Station": "sta",
|
||
"Stravenue": "stra",
|
||
"Stream": "strm",
|
||
"Street": "st",
|
||
"Summit": "smt",
|
||
"Terrace": "ter",
|
||
"Throughway": "trwy",
|
||
"Trace": "trce",
|
||
"Track": "trak",
|
||
"Trafficway": "trfy",
|
||
"Trail": "trl",
|
||
"Trailer": "trlr",
|
||
"Tunnel": "tunl",
|
||
"Turnpike": "tpke",
|
||
"Underpass": "upas",
|
||
"Union": "un",
|
||
"Valley": "vly",
|
||
"Viaduct": "via",
|
||
"View": "vw",
|
||
"Village": "vlg",
|
||
"Ville": "vl",
|
||
"Vista": "vis",
|
||
"Walk": "walk",
|
||
"Way": "way",
|
||
"Well": "wl",
|
||
"Wells": "wls",
|
||
"Wye": "wye"
|
||
}
|
||
|
||
const app = express()
|
||
const port = 3000
|
||
|
||
const hlrToken = ''
|
||
const hlrSecret = ''
|
||
const basic = crypto.createHash('sha256').update(hlrToken + ':' + hlrSecret).digest('hex');
|
||
|
||
const uuidRegex = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/;
|
||
|
||
let handledCharges: string[] = []
|
||
|
||
// parse application/x-www-form-urlencoded
|
||
app.use(bodyParser.urlencoded({ extended: false }))
|
||
|
||
// parse application/json
|
||
app.use(bodyParser.json())
|
||
|
||
const fs = require('fs')
|
||
|
||
const donations = JSON.parse(fs.readFileSync('templates/pages/donations.json', 'utf-8'))
|
||
|
||
|
||
// const mapTemplate = fs.readFileSync('templates/pages/map.html', 'utf-8')
|
||
const indexTemplate = fs.readFileSync('templates/pages/home.mustache', 'utf-8')
|
||
const recordByIdTemplate = fs.readFileSync('templates/pages/recordById.mustache', 'utf-8')
|
||
const recordsListingTemplate = fs.readFileSync('templates/pages/recordsListing.mustache', 'utf-8')
|
||
const donationsTemplate = fs.readFileSync('templates/pages/donations.html', 'utf-8')
|
||
const exportsTemplate = fs.readFileSync('templates/pages/exports.html', 'utf-8')
|
||
|
||
const visualizerTemplate = fs.readFileSync('templates/pages/visualizer.html', 'utf-8')
|
||
|
||
|
||
const servers = ["http://solr1:8983/solr/BigData/select"]
|
||
|
||
|
||
const blacklistedAutomatedIps: string[] = []
|
||
const whitelistedUserAgents: string[] = [
|
||
// API Keys
|
||
]
|
||
|
||
// Stores that last time a query was made to a server for an IP address
|
||
const lastQueryTime: Record<string, number> = {}
|
||
const lastQueryTimes: Record<string, number[]> = {}
|
||
const numLookupsPerIP: Record<string, number> = {}
|
||
let hasRequestedWallet: string[] = []
|
||
|
||
function getRandServer() {
|
||
var index = Math.floor(Math.random() * servers.length);
|
||
return servers[index];
|
||
}
|
||
|
||
function getRandWalletsServer() {
|
||
var index = Math.floor(Math.random() * servers.length);
|
||
return servers[index].replace('BigData', 'Wallets');
|
||
}
|
||
|
||
|
||
|
||
async function queryForDocsSpatial(latLong: string, distanceKm: number) {
|
||
const records: SolrRecord[] = []
|
||
let numDocs = 0
|
||
|
||
await axios.get(getRandServer(), {
|
||
params: {
|
||
q: "latLong:*",
|
||
rows: 500,
|
||
fq: "{!geofilt sfield=latLong}",
|
||
pt: latLong,
|
||
d: distanceKm
|
||
}
|
||
}).then(({ data }) => {
|
||
records.push(...data.response.docs)
|
||
numDocs += data.response.numFound
|
||
})
|
||
|
||
console.log(`Spatial query for ${latLong} returned ${numDocs} records.`)
|
||
|
||
return {
|
||
numDocs: numDocs,
|
||
records: records
|
||
}
|
||
}
|
||
|
||
|
||
async function getWalletBalance(wallet: string): Promise<number> {
|
||
return axios.get(getRandWalletsServer(), {
|
||
params: {
|
||
q: `id:"${wallet}"`
|
||
}
|
||
})
|
||
.then(({ data }) => {
|
||
console.log(data.response.docs[0])
|
||
if (data.response.docs[0]) {
|
||
return data.response.docs[0].credits
|
||
} else {
|
||
return 0
|
||
}
|
||
}
|
||
)
|
||
.catch((err) => { console.log(err); return 0 })
|
||
}
|
||
|
||
async function addWalletBalance(wallet: string, credits: number) {
|
||
if (typeof credits !== 'number') {
|
||
console.log(`Credits is not a number: ${credits}`)
|
||
throw new Error('Credits is not a number')
|
||
}
|
||
|
||
// Get the old balance
|
||
const oldBalance = await getWalletBalance(wallet)
|
||
|
||
console.log('old balance: ' + oldBalance + ' credits: ' + credits + ' wallet: ' + wallet + ' new balance: ' + (oldBalance + credits))
|
||
// Add the new balance
|
||
credits += oldBalance
|
||
// Update the wallet
|
||
return axios.post(getRandWalletsServer().replace('select', 'update'), {
|
||
"add": {
|
||
"doc": {
|
||
// uuid v4
|
||
"id": wallet,
|
||
"credits": credits
|
||
}
|
||
}
|
||
}, {
|
||
params: {
|
||
commit: true
|
||
}
|
||
}).then(() => {
|
||
console.log(`Added ${credits} credits to wallet ${wallet}`)
|
||
return true
|
||
}) .catch ((err) => {
|
||
console.log(err)
|
||
return false
|
||
})
|
||
}
|
||
|
||
async function removeWalletBalance(wallet: string, credits: number) {
|
||
if (typeof credits !== 'number') {
|
||
console.log(`Credits is not a number: ${credits}`)
|
||
throw new Error('Credits is not a number')
|
||
}
|
||
|
||
// Get the old balance
|
||
const oldBalance = await getWalletBalance(wallet)
|
||
// Remove the new balance
|
||
const newBalance = oldBalance - credits
|
||
// Update the wallet
|
||
return axios.post(getRandWalletsServer().replace('select', 'update'), {
|
||
"add": {
|
||
"doc": {
|
||
// uuid v4
|
||
"id": wallet,
|
||
"credits": newBalance
|
||
}
|
||
}
|
||
}, {
|
||
params: {
|
||
commit: true
|
||
}
|
||
}).then(() => {
|
||
console.log(`Removed ${credits} credits to wallet ${wallet}`)
|
||
return true
|
||
})
|
||
}
|
||
|
||
async function queryForDocs(query: string, limit?: number, start?: number, sort?: string) {
|
||
const records: SolrRecord[] = []
|
||
let numDocs = 0
|
||
|
||
await axios.get(getRandServer(), {
|
||
params: {
|
||
q: query,
|
||
rows: limit || 100,
|
||
sort: sort,
|
||
start: start || 0
|
||
}
|
||
}).then(({ data }) => {
|
||
records.push(...data.response.docs)
|
||
numDocs += data.response.numFound
|
||
})
|
||
|
||
|
||
return {
|
||
numDocs: numDocs,
|
||
records: records
|
||
}
|
||
}
|
||
|
||
async function queryForExportDocs(query: string, limit?: number, start?: number, sort?: string) {
|
||
const records: SolrRecord[] = []
|
||
let numDocs = 0
|
||
|
||
await axios.get(getRandServer().replace('BigData', 'Exports'), {
|
||
params: {
|
||
q: query,
|
||
rows: limit || 100,
|
||
sort: sort,
|
||
start: start || 0
|
||
}
|
||
}).then(({ data }) => {
|
||
records.push(...data.response.docs)
|
||
numDocs += data.response.numFound
|
||
})
|
||
|
||
return {
|
||
numDocs: numDocs,
|
||
records: records
|
||
}
|
||
}
|
||
|
||
function buildQuery(requestedQuery: Record<string, string | string[]>, res: express.Response) {
|
||
let query: string[] = []
|
||
let orQuery: string[] = []
|
||
let notQuery: string[] = []
|
||
let additionalQuery: string[] = []
|
||
|
||
|
||
let doAdditionalQuery = false
|
||
|
||
|
||
if (requestedQuery.firstName != undefined) {
|
||
if (typeof requestedQuery.firstName == 'string') {
|
||
query.push(`firstName:${requestedQuery.firstName.replace(' ', '?')}`)
|
||
}
|
||
}
|
||
if (requestedQuery.notfirstName != undefined) {
|
||
if (typeof requestedQuery.notfirstName === 'string') {
|
||
notQuery.push(`firstName:${requestedQuery.notfirstName.replace(' ', '?')}`)
|
||
} else if (Array.isArray(requestedQuery.notfirstName)) {
|
||
requestedQuery.notfirstName.forEach((notItem: any) => {
|
||
notQuery.push(`firstName:${notItem.replace(' ', '?')}`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.lastName != undefined) {
|
||
if (typeof requestedQuery.lastName == 'string') {
|
||
query.push(`lastName:${requestedQuery.lastName.replace(' ', '?')}`)
|
||
}
|
||
}
|
||
if (requestedQuery.notlastName != undefined) {
|
||
if (typeof requestedQuery.notlastName === 'string') {
|
||
notQuery.push(`lastName:${requestedQuery.notlastName.replace(' ', '?')}`)
|
||
} else if (Array.isArray(requestedQuery.notlastName)) {
|
||
requestedQuery.notlastName.forEach((notItem: any) => {
|
||
notQuery.push(`lastName:${notItem.replace(' ', '?')}`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.birthYear != undefined) {
|
||
if (typeof requestedQuery.birthYear == 'string') {
|
||
query.push(`birthYear:${requestedQuery.birthYear}`)
|
||
}
|
||
}
|
||
if (requestedQuery.notbirthYear != undefined) {
|
||
if (typeof requestedQuery.notbirthYear === 'string') {
|
||
notQuery.push(`birthYear:${requestedQuery.notbirthYear.replace(' ', '?')}`)
|
||
} else if (Array.isArray(requestedQuery.notbirthYear)) {
|
||
requestedQuery.notbirthYear.forEach((notItem: any) => {
|
||
notQuery.push(`birthYear:${notItem.replace(' ', '?')}`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.ips != undefined) {
|
||
if (typeof requestedQuery.ips == 'string') {
|
||
query.push(`ips:WRfKdFVogXnk82${requestedQuery.ips}WRfKdFVogXnk82`)
|
||
}
|
||
}
|
||
if (requestedQuery.notips != undefined) {
|
||
if (typeof requestedQuery.notips === 'string') {
|
||
notQuery.push(`ips:WRfKdFVogXnk82${requestedQuery.notips}WRfKdFVogXnk82`)
|
||
} else if (Array.isArray(requestedQuery.notips)) {
|
||
requestedQuery.notips.forEach((notItem: any) => {
|
||
notQuery.push(`ips:WRfKdFVogXnk82${notItem}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.asn != undefined) {
|
||
if (typeof requestedQuery.asn == 'string') {
|
||
query.push(`asn:${requestedQuery.asn}`)
|
||
}
|
||
}
|
||
if (requestedQuery.notasn != undefined) {
|
||
if (typeof requestedQuery.notasn === 'string') {
|
||
notQuery.push(`asn:${requestedQuery.notasn.replace(' ', '?')}`)
|
||
} else if (Array.isArray(requestedQuery.notasn)) {
|
||
requestedQuery.notasn.forEach((notItem: any) => {
|
||
notQuery.push(`asn:${notItem.replace(' ', '?')}`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.domain != undefined) {
|
||
if (requestedQuery.exact) {
|
||
if (typeof requestedQuery.domain == 'string') {
|
||
query.push(`domain:WRfKdFVogXnk82${requestedQuery.domain.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
}
|
||
} else {
|
||
if (typeof requestedQuery.domain == 'string') {
|
||
query.push(`domain:${requestedQuery.domain.replace(' ', '?')}`)
|
||
}
|
||
}
|
||
|
||
}
|
||
if (requestedQuery.notdomain != undefined) {
|
||
if (typeof requestedQuery.notdomain === 'string') {
|
||
notQuery.push(`domain:WRfKdFVogXnk82${requestedQuery.notdomain.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
} else if (Array.isArray(requestedQuery.notdomain)) {
|
||
requestedQuery.notdomain.forEach((notItem: any) => {
|
||
notQuery.push(`domain:WRfKdFVogXnk82${notItem.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
|
||
if (requestedQuery.asnOrg != undefined) {
|
||
if (typeof requestedQuery.asnOrg == 'string') {
|
||
query.push(`asnOrg:WRfKdFVogXnk82${requestedQuery.asnOrg}WRfKdFVogXnk82`)
|
||
}
|
||
}
|
||
if (requestedQuery.notasnOrg != undefined) {
|
||
if (typeof requestedQuery.notasnOrg === 'string') {
|
||
notQuery.push(`asnOrg:${requestedQuery.notasnOrg.replace(' ', '?')}`)
|
||
} else if (Array.isArray(requestedQuery.notasnOrg)) {
|
||
requestedQuery.notasnOrg.forEach((notItem: any) => {
|
||
notQuery.push(`asnOrg:WRfKdFVogXnk82${notItem.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.country != undefined) {
|
||
if (typeof requestedQuery.country == 'string') {
|
||
query.push(`country:WRfKdFVogXnk82${requestedQuery.country}WRfKdFVogXnk82`)
|
||
}
|
||
}
|
||
if (requestedQuery.notcountry != undefined) {
|
||
if (typeof requestedQuery.notcountry === 'string') {
|
||
notQuery.push(`country:WRfKdFVogXnk82${requestedQuery.notcountry.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
} else if (Array.isArray(requestedQuery.notcountry)) {
|
||
requestedQuery.notcountry.forEach((notItem: any) => {
|
||
notQuery.push(`country:WRfKdFVogXnk82${notItem.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.continent != undefined) {
|
||
if (typeof requestedQuery.continent == 'string') {
|
||
query.push(`continent:WRfKdFVogXnk82${requestedQuery.continent}WRfKdFVogXnk82`)
|
||
}
|
||
}
|
||
if (requestedQuery.notcontinent != undefined) {
|
||
if (typeof requestedQuery.notcontinent === 'string') {
|
||
notQuery.push(`continent:WRfKdFVogXnk82${requestedQuery.notcontinent.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
} else if (Array.isArray(requestedQuery.notcontinent)) {
|
||
requestedQuery.notcontinent.forEach((notItem: any) => {
|
||
notQuery.push(`continent:WRfKdFVogXnk82${notItem.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
|
||
|
||
if (requestedQuery.firstName != undefined && requestedQuery.lastName != undefined && typeof requestedQuery.firstName == 'string' && typeof requestedQuery.lastName == 'string') {
|
||
const firstName = requestedQuery.firstName.replace(' ', '?')
|
||
const lastName = requestedQuery.lastName.replace(' ', '?')
|
||
|
||
if (!requestedQuery.exact) {
|
||
additionalQuery.push(`emails:${firstName}?${lastName}`)
|
||
doAdditionalQuery = true
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.source != undefined) {
|
||
if (typeof requestedQuery.source == 'string') {
|
||
query.push(`source:WRfKdFVogXnk82${requestedQuery.source}WRfKdFVogXnk82`)
|
||
}
|
||
}
|
||
if (requestedQuery.notsource != undefined) {
|
||
if (typeof requestedQuery.notsource === 'string') {
|
||
notQuery.push(`source:WRfKdFVogXnk82${requestedQuery.notsource.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
} else if (Array.isArray(requestedQuery.notsource)) {
|
||
requestedQuery.notsource.forEach((notItem: any) => {
|
||
notQuery.push(`source:WRfKdFVogXnk82${notItem.replace(' ', '?')}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.emails != undefined) {
|
||
if (typeof requestedQuery.emails == 'string') {
|
||
|
||
// check if there's a * in the email
|
||
if (requestedQuery.emails.indexOf('*') != -1) {
|
||
query.push(`emails:${requestedQuery.emails.replace('@', '?')}`)
|
||
}
|
||
else {
|
||
if (requestedQuery.exact) {
|
||
query.push(`emails:WRfKdFVogXnk82${requestedQuery.emails.replace('@', '?')}WRfKdFVogXnk82`)
|
||
} else {
|
||
query.push(`emails:${requestedQuery.emails.replace('@', '?')}`)
|
||
}
|
||
|
||
}
|
||
|
||
|
||
const mostCommonEmailDomains = [/\@gmail\..*/gi, /\@yahoo\..*/gi, /\@hotmail\..*/gi, /\@outlook\..*/gi, /\@aol\..*/gi, /\@icloud\..*/gi, /\@mail\..*/gi, /\@protonmail\..*/gi, /\@zoho\..*/gi, /\@msn\..*/gi, /\@yandex\..*/gi, /\@gmx\..*/gi, /\@live\..*/gi, /\@mail\.ru\..*/gi, /\@inbox\..*/gi, /\@ymail\..*/gi, /\@comcast\..*/gi, /\@verizon\..*/gi, /\@att\..*/gi, /\@sbcglobal\..*/gi, /\@cox\..*/gi, /\@earthlink\..*/gi, /\@charter\..*/gi, /\@optonline\..*/gi, /\@frontier\..*/gi, /\@windstream\..*/gi, /\@q\.com\..*/gi, /\@btinternet\..*/gi, /\@btconnect\..*/gi, /\@ntlworld\..*/gi, /\@bt\..*/gi, /\@virginmedia\..*/gi, /\@btopenworld\..*/gi, /\@talktalk\..*/gi, /\@sky\..*/gi, /\@orange\..*/gi, /\@bt\..*/gi, /\@virgin\..*/gi, /\@ntl\..*/gi, /\@freeserve\..*/gi, /\@blueyonder\..*/gi, /\@btinternet\..*/gi, /\@tiscali\..*/gi, /\@virgin\..*/gi, /\@tesco\..*/gi, /\@onetel\..*/gi, /\@bt\..*/gi, /\@virgin\..*/gi, /\@ntl\..*/gi, /\@freeserve\..*/gi, /\@blueyonder\..*/gi, /\@btinternet\..*/gi, /\@tiscali\..*/gi, /\@virgin\..*/gi, /\@tesco\..*/gi, /\@onetel\..*/gi, /\@bt\..*/gi, /\@virgin\..*/]
|
||
const emailSplit = requestedQuery.emails.split('@')
|
||
|
||
if (requestedQuery.emails.indexOf('*') == -1) {
|
||
additionalQuery.push(`emails:WRfKdFVogXnk82${emailSplit[0]}WRfKdFVogXnk82`)
|
||
}
|
||
|
||
|
||
doAdditionalQuery = true;
|
||
|
||
let isCommonDomain = false;
|
||
|
||
for (const domain of mostCommonEmailDomains) {
|
||
if (domain.test(requestedQuery.emails as string)) {
|
||
isCommonDomain = true
|
||
}
|
||
}
|
||
|
||
if (!isCommonDomain) {
|
||
if (emailSplit.length === 2) {
|
||
const domain = emailSplit[emailSplit.length - 1]
|
||
additionalQuery.push(`emails:${domain}`)
|
||
doAdditionalQuery = true
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.notemails != undefined) {
|
||
if (typeof requestedQuery.notemails === 'string') {
|
||
notQuery.push(`emails:WRfKdFVogXnk82${requestedQuery.notemails.replace('@', '?')}WRfKdFVogXnk82`)
|
||
} else if (Array.isArray(requestedQuery.notemails)) {
|
||
requestedQuery.notemails.forEach((notItem: any) => {
|
||
notQuery.push(`emails:WRfKdFVogXnk82${notItem.replace('@', '?')}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.VRN != undefined) {
|
||
if (typeof requestedQuery.VRN == 'string') {
|
||
query.push(`VRN:${requestedQuery.VRN.toLowerCase()}`)
|
||
}
|
||
}
|
||
if (requestedQuery.notVRN != undefined) {
|
||
if (typeof requestedQuery.notVRN === 'string') {
|
||
notQuery.push(`VRN:${requestedQuery.notVRN.toLowerCase()}`)
|
||
} else if (Array.isArray(requestedQuery.notVRN)) {
|
||
requestedQuery.notVRN.forEach((notItem: any) => {
|
||
notQuery.push(`VRN:${notItem.toLowerCase()}`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.usernames != undefined) {
|
||
if (requestedQuery.exact) {
|
||
query.push(`usernames:WRfKdFVogXnk82${requestedQuery.usernames}WRfKdFVogXnk82`)
|
||
} else {
|
||
query.push(`usernames:${requestedQuery.usernames}`)
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.notusernames != undefined) {
|
||
if (typeof requestedQuery.notusernames === 'string') {
|
||
notQuery.push(`usernames:${requestedQuery.notusernames}`)
|
||
} else if (Array.isArray(requestedQuery.notusernames)) {
|
||
requestedQuery.notusernames.forEach((notItem: any) => {
|
||
notQuery.push(`usernames:${notItem.toLowerCase()}`)
|
||
})
|
||
}
|
||
}
|
||
|
||
|
||
if (requestedQuery.address != undefined) {
|
||
query.push(`address:WRfKdFVogXnk82${requestedQuery.address}WRfKdFVogXnk82`)
|
||
}
|
||
if (requestedQuery.notaddress != undefined) {
|
||
if (typeof requestedQuery.notaddress === 'string') {
|
||
notQuery.push(`address:WRfKdFVogXnk82${requestedQuery.notaddress}WRfKdFVogXnk82`)
|
||
} else if (Array.isArray(requestedQuery.notaddress)) {
|
||
requestedQuery.notaddress.forEach((notItem: any) => {
|
||
notQuery.push(`address:WRfKdFVogXnk82${notItem}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.city != undefined) {
|
||
query.push(`city:${requestedQuery.city}`)
|
||
}
|
||
|
||
if (requestedQuery.notcity != undefined) {
|
||
if (typeof requestedQuery.notcity === 'string') {
|
||
notQuery.push(`city:WRfKdFVogXnk82${requestedQuery.notcity}WRfKdFVogXnk82`)
|
||
} else if (Array.isArray(requestedQuery.notcity)) {
|
||
requestedQuery.notcity.forEach((notItem: any) => {
|
||
notQuery.push(`city:WRfKdFVogXnk82${notItem}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.zipCode != undefined) {
|
||
query.push(`zipCode:${requestedQuery.zipCode}`)
|
||
}
|
||
if (requestedQuery.notzipCode != undefined) {
|
||
if (typeof requestedQuery.notzipCode === 'string') {
|
||
notQuery.push(`address:WRfKdFVogXnk82${requestedQuery.notzipCode}WRfKdFVogXnk82`)
|
||
} else if (Array.isArray(requestedQuery.notzipCode)) {
|
||
requestedQuery.notzipCode.forEach((notItem: any) => {
|
||
notQuery.push(`address:WRfKdFVogXnk82${notItem}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.state != undefined) {
|
||
query.push(`state:${requestedQuery.state}`)
|
||
}
|
||
if (requestedQuery.notstate != undefined) {
|
||
if (typeof requestedQuery.notstate === 'string') {
|
||
notQuery.push(`state:${requestedQuery.notstate}`)
|
||
} else if (Array.isArray(requestedQuery.notstate)) {
|
||
requestedQuery.notstate.forEach((notItem: any) => {
|
||
notQuery.push(`state:${notItem}`)
|
||
})
|
||
}
|
||
}
|
||
|
||
|
||
if (requestedQuery.phoneNumbers != undefined && typeof requestedQuery.phoneNumbers == 'string') {
|
||
|
||
if (!requestedQuery.exact) {
|
||
query.push(`phoneNumbers:WRfKdFVogXnk82${requestedQuery.phoneNumbers.replace(/\D+/g, "")}WRfKdFVogXnk82`)
|
||
} else {
|
||
// Replace all non-digits or non-question marks with
|
||
query.push(`phoneNumbers:WRfKdFVogXnk82${requestedQuery.phoneNumbers.replace(/\D+/g, "")}WRfKdFVogXnk82`)
|
||
additionalQuery.push(`phoneNumbers:1${requestedQuery.phoneNumbers.replace(/\D+/g, "")}`)
|
||
additionalQuery.push(`phoneNumbers:7${requestedQuery.phoneNumbers.replace(/\D+/g, "")}`)
|
||
doAdditionalQuery = true;
|
||
}
|
||
|
||
}
|
||
|
||
if (requestedQuery.notphoneNumbers != undefined) {
|
||
if (typeof requestedQuery.notphoneNumbers === 'string') {
|
||
notQuery.push(`phoneNumbers:${requestedQuery.notphoneNumbers}`)
|
||
} else if (Array.isArray(requestedQuery.notphoneNumbers)) {
|
||
requestedQuery.notphoneNumbers.forEach((notItem: any) => {
|
||
notQuery.push(`phoneNumbers:${notItem}`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.passwords != undefined && typeof requestedQuery.passwords === 'string') {
|
||
query.push(`passwords:${requestedQuery.passwords}`)
|
||
} else if (requestedQuery.passwords != undefined && Array.isArray(requestedQuery.passwords)) {
|
||
requestedQuery.passwords.forEach((password) => {
|
||
orQuery.push(`passwords:WRfKdFVogXnk82${password}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
|
||
if (requestedQuery.notpasswords != undefined) {
|
||
if (typeof requestedQuery.notpasswords === 'string') {
|
||
notQuery.push(`passwords:${requestedQuery.notpasswords}`)
|
||
} else if (Array.isArray(requestedQuery.notpasswords)) {
|
||
requestedQuery.notpasswords.forEach((notItem: any) => {
|
||
notQuery.push(`passwords:${notItem}`)
|
||
})
|
||
}
|
||
}
|
||
|
||
if (requestedQuery.vin != undefined && typeof requestedQuery.vin === 'string') {
|
||
query.push(`vin:${requestedQuery.vin}`)
|
||
} else if (requestedQuery.vin != undefined && Array.isArray(requestedQuery.vin)) {
|
||
requestedQuery.vin.forEach((vin) => {
|
||
orQuery.push(`vin:WRfKdFVogXnk82${vin}WRfKdFVogXnk82`)
|
||
})
|
||
}
|
||
|
||
|
||
|
||
let queryBuilt = sanitizeQuery(query.join(' AND '))
|
||
|
||
if (orQuery.length > 0) {
|
||
if (query.length > 0) {
|
||
queryBuilt += ' OR ' + sanitizeQuery(orQuery.join(' OR '))
|
||
} else {
|
||
queryBuilt += sanitizeQuery(orQuery.join(' OR '))
|
||
}
|
||
}
|
||
|
||
if (notQuery.length > 0) {
|
||
if (query.length > 0) {
|
||
queryBuilt += ' NOT ' + sanitizeQuery(notQuery.join(' NOT '))
|
||
} else {
|
||
return res.status(400).send("Error: you sent an negative query without a regular query! Silly goose.")
|
||
}
|
||
}
|
||
|
||
return {
|
||
query: queryBuilt,
|
||
additionalQuery: additionalQuery,
|
||
doAdditionalQuery: doAdditionalQuery
|
||
}
|
||
}
|
||
|
||
async function getSimilarRecords(record: SolrRecord) {
|
||
let responses: any[] = []
|
||
|
||
const scoresList: Record<string, number> = {}
|
||
|
||
const relatedDocIDs: string[] = []
|
||
|
||
await Promise.all([
|
||
...(record.emails || []).map((email) => {
|
||
return axios.get(getRandServer(), {
|
||
params: {
|
||
q: `emails:"${email.replace(/\"/gi, '').replace(/\//gi, '')}"`,
|
||
rows: 20,
|
||
fl: 'id'
|
||
}
|
||
}).then(({ data }) => {
|
||
data.response.docs.forEach((doc: SolrRecord) => {
|
||
if (doc.id === record.id) {
|
||
return
|
||
}
|
||
if (!(doc.id in scoresList)) {
|
||
scoresList[doc.id] = 15
|
||
} else {
|
||
scoresList[doc.id] += 5
|
||
}
|
||
relatedDocIDs.push(doc.id)
|
||
})
|
||
})
|
||
}),
|
||
...(record.usernames || []).map((email) => {
|
||
return axios.get(getRandServer(), {
|
||
params: {
|
||
q: `usernames:"${email.replace(/\"/gi, '').replace(/\//gi, '')}"`,
|
||
rows: 20,
|
||
fl: 'id'
|
||
}
|
||
}).then(({ data }) => {
|
||
data.response.docs.forEach((doc: SolrRecord) => {
|
||
if (doc.id === record.id) {
|
||
return
|
||
}
|
||
if (!(doc.id in scoresList)) {
|
||
scoresList[doc.id] = 15
|
||
} else {
|
||
scoresList[doc.id] += 5
|
||
}
|
||
relatedDocIDs.push(doc.id)
|
||
})
|
||
})
|
||
}),
|
||
axios.get(getRandServer(), {
|
||
params: {
|
||
q: `id:${record.id}`,
|
||
mlt: true,
|
||
'mlt.fl': 'emails',
|
||
'mlt.mindf': 2,
|
||
'mlt.mintf': 2,
|
||
}
|
||
}).then(({ data }) => {
|
||
data.moreLikeThis.forEach((doc: any) => {
|
||
if (typeof doc == 'object') {
|
||
doc.docs.forEach((relatedDoc: any) => {
|
||
if (scoresList[relatedDoc.id]) {
|
||
scoresList[relatedDoc.id] += relatedDoc.score
|
||
} else {
|
||
scoresList[relatedDoc.id] = relatedDoc.score
|
||
|
||
}
|
||
})
|
||
}
|
||
})
|
||
responses.push(...data.moreLikeThis)
|
||
}),
|
||
axios.get(getRandServer(), {
|
||
params: {
|
||
q: `id:${record.id}`,
|
||
mlt: true,
|
||
'mlt.fl': 'usernames',
|
||
'mlt.mindf': 1,
|
||
'mlt.mintf': 1,
|
||
'mlt.match.include': true,
|
||
}
|
||
}).then(({ data }) => {
|
||
data.moreLikeThis.forEach((doc: any) => {
|
||
if (typeof doc == 'object') {
|
||
doc.docs.forEach((relatedDoc: any) => {
|
||
if (scoresList[relatedDoc.id]) {
|
||
scoresList[relatedDoc.id] += relatedDoc.score
|
||
} else {
|
||
scoresList[relatedDoc.id] = relatedDoc.score
|
||
|
||
}
|
||
})
|
||
}
|
||
})
|
||
responses.push(...data.moreLikeThis)
|
||
}),
|
||
axios.get(getRandServer(), {
|
||
params: {
|
||
q: `id:${record.id}`,
|
||
mlt: true,
|
||
'mlt.fl': 'phoneNumbers',
|
||
'mlt.mindf': 1,
|
||
'mlt.mintf': 1,
|
||
'mlt.match.include': true,
|
||
}
|
||
}).then(({ data }) => {
|
||
data.moreLikeThis.forEach((doc: any) => {
|
||
if (typeof doc == 'object') {
|
||
doc.docs.forEach((relatedDoc: any) => {
|
||
if (scoresList[relatedDoc.id]) {
|
||
scoresList[relatedDoc.id] += relatedDoc.score
|
||
} else {
|
||
scoresList[relatedDoc.id] = relatedDoc.score
|
||
|
||
}
|
||
})
|
||
}
|
||
})
|
||
responses.push(...data.moreLikeThis)
|
||
}),
|
||
axios.get(getRandServer(), {
|
||
params: {
|
||
q: `id:${record.id}`,
|
||
mlt: true,
|
||
'mlt.fl': 'address_search',
|
||
'mlt.mindf': 1,
|
||
'mlt.mintf': 1,
|
||
'mlt.match.include': true,
|
||
}
|
||
}).then(({ data }) => {
|
||
data.moreLikeThis.forEach((doc: any) => {
|
||
if (typeof doc == 'object') {
|
||
doc.docs.forEach((relatedDoc: any) => {
|
||
if (scoresList[relatedDoc.id]) {
|
||
scoresList[relatedDoc.id] += relatedDoc.score
|
||
} else {
|
||
scoresList[relatedDoc.id] = relatedDoc.score
|
||
|
||
}
|
||
})
|
||
}
|
||
})
|
||
responses.push(...data.moreLikeThis)
|
||
}),
|
||
axios.get(getRandServer(), {
|
||
params: {
|
||
q: `id:${record.id}`,
|
||
mlt: true,
|
||
'mlt.fl': 'firstName,lastName',
|
||
'mlt.mindf': 1,
|
||
'mlt.mintf': 1,
|
||
'mlt.match.include': true,
|
||
}
|
||
}).then(({ data }) => {
|
||
data.moreLikeThis.forEach((doc: any) => {
|
||
if (typeof doc == 'object') {
|
||
doc.docs.forEach((relatedDoc: any) => {
|
||
if (scoresList[relatedDoc.id]) {
|
||
scoresList[relatedDoc.id] += relatedDoc.score
|
||
} else {
|
||
scoresList[relatedDoc.id] = relatedDoc.score
|
||
}
|
||
})
|
||
}
|
||
})
|
||
responses.push(...data.moreLikeThis)
|
||
}),
|
||
])
|
||
|
||
|
||
responses.forEach((response) => {
|
||
if (typeof response == 'object') {
|
||
relatedDocIDs.push(...(response.docs.map((doc: any) => doc.id)))
|
||
}
|
||
})
|
||
|
||
let docs: SolrRecord[] = []
|
||
|
||
// Get unique doc IDs
|
||
const uniqueDocIDs = [...new Set(relatedDocIDs)]
|
||
|
||
|
||
|
||
await Promise.all(uniqueDocIDs.map((doc: any) => queryForDocs(`id:${doc}`))).then((results) => {
|
||
results.forEach((result) => {
|
||
docs.push(...result.records.map((result) => {
|
||
return {
|
||
...result,
|
||
'similarity score': scoresList[result.id] || 0,
|
||
}
|
||
}))
|
||
})
|
||
})
|
||
|
||
// Sort the docs by score
|
||
docs = docs.sort((a, b) => {
|
||
if (scoresList[a.id] > scoresList[b.id]) {
|
||
return -1
|
||
} else if (scoresList[a.id] < scoresList[b.id]) {
|
||
return 1
|
||
} else {
|
||
return 0
|
||
}
|
||
})
|
||
|
||
return docs
|
||
}
|
||
|
||
function sanitizeQuery(query: string) {
|
||
//return query;
|
||
return query.replace(/[^\w\s\$\:\.\@\-\*\а-яА-ЯёЁ]/gi, '?').replace(/WRfKdFVogXnk82/g, '"')
|
||
}
|
||
|
||
app.get('/exports', async (req, res) => {
|
||
const rendered = mustache.render(exportsTemplate, {})
|
||
return res.send(rendered)
|
||
})
|
||
|
||
app.get('/exports/:walletid', async (req, res) => {
|
||
try {
|
||
const walletid = req.params.walletid
|
||
// Ensure the wallet ID is valid
|
||
if (!uuidRegex.test(walletid)) {
|
||
return res.status(400).json({
|
||
error: 'Invalid wallet ID.'
|
||
})
|
||
}
|
||
const records = await queryForExportDocs(`wallet:"${walletid}"`)
|
||
|
||
return res.json(records.records.map((record: any) => {
|
||
return {
|
||
...record,
|
||
jobid: record.id,
|
||
exportCount: record.count,
|
||
}
|
||
}))
|
||
} catch (e) {
|
||
return res.status(500).json([
|
||
{
|
||
id: 'internal error',
|
||
query: 'sorry bout that... try again?'
|
||
}
|
||
])
|
||
}
|
||
|
||
})
|
||
|
||
app.get('/', async (req, res) => {
|
||
const rendered = mustache.render(indexTemplate, { count: "14,491,682,918" })
|
||
|
||
return res.send(rendered)
|
||
})
|
||
|
||
|
||
app.get('/visualize', async (req, res) => {
|
||
const rendered = mustache.render(visualizerTemplate, {})
|
||
|
||
return res.send(rendered)
|
||
})
|
||
|
||
app.get('/map', async (req, res) => {
|
||
res.sendFile(__dirname + '/static/map.html')
|
||
})
|
||
|
||
app.get('/favicon.ico', (req, res) => {
|
||
// Read the favicon file and send it to the client
|
||
res.sendFile(__dirname + '/static/favicon.ico')
|
||
})
|
||
|
||
app.get('/robots.txt', (req, res) => {
|
||
// Read robots.txt and send it to the client
|
||
res.sendFile(__dirname + '/static/robots.txt')
|
||
})
|
||
|
||
app.get('/dear_peter', (req, res) => {
|
||
res.sendFile(__dirname + '/static/dear_peter.html')
|
||
})
|
||
|
||
app.get('/terms', (req, res) => {
|
||
res.sendFile(__dirname + '/static/terms.html')
|
||
})
|
||
|
||
app.get('/faq', (req, res) => {
|
||
res.sendFile(__dirname + '/static/faq.html')
|
||
})
|
||
|
||
app.get('/canary', (req, res) => {
|
||
res.sendFile(__dirname + '/static/canary.html')
|
||
})
|
||
|
||
app.get('/donations', (req, res) => {
|
||
// Render out the donations page tempate
|
||
const rendered = mustache.render(donationsTemplate, {
|
||
donations: donations.sort((a: any, b: any) => b.amount - a.amount)
|
||
})
|
||
|
||
return res.send(rendered)
|
||
})
|
||
|
||
app.post('/export', async (req, res) => {
|
||
try {
|
||
// Read body
|
||
let body = req.body
|
||
|
||
// Check if the body is valid
|
||
if (!body || !body.walletId || !body.exportCount) {
|
||
return res.status(400).send('Invalid body')
|
||
}
|
||
|
||
// Make sure export count is > 0
|
||
if (typeof body.exportCount !== 'number') {
|
||
// Set it to be a number
|
||
body.exportCount = parseInt(body.exportCount, 10)
|
||
}
|
||
if (body.exportCount <= 0) {
|
||
return res.status(400).send('Invalid export count')
|
||
}
|
||
|
||
// If the wallet ID is invalid uuid4
|
||
if (!uuidRegex.test(body.walletId)) {
|
||
return res.status(400).send('Invalid wallet ID')
|
||
}
|
||
|
||
// Get credits for the wallet
|
||
const credits = await getWalletBalance(body.walletId)
|
||
|
||
const cost = (body.exportCount - 100) / 10
|
||
|
||
if (cost > credits) {
|
||
return res.status(400).send('Insufficient credits')
|
||
}
|
||
|
||
// if cost < 0
|
||
if (cost < 0) {
|
||
return res.status(400).send('Invalid export count')
|
||
}
|
||
|
||
|
||
let requestedQuery: Record<string, any> = {}
|
||
|
||
for (const [key, value] of Object.entries(req.query)) {
|
||
if (key !== 'wt') {
|
||
if (typeof value === 'string' || Array.isArray(value)) {
|
||
requestedQuery[key] = value
|
||
} else {
|
||
console.log(`Invalid query: ${key} is not a string or array. ${value}`)
|
||
}
|
||
}
|
||
}
|
||
|
||
const queryBuilt = buildQuery(requestedQuery, res as any) as any
|
||
|
||
const jobid = uuidv4()
|
||
|
||
if (!queryBuilt.query || !queryBuilt.additionalQuery || typeof queryBuilt.doAdditionalQuery !== 'boolean') {
|
||
return res.status(400).send('Invalid query')
|
||
}
|
||
|
||
let finalQuery = queryBuilt.query
|
||
|
||
if (queryBuilt.additionalQuery.length > 0) {
|
||
finalQuery = `(${finalQuery}) OR ${sanitizeQuery(queryBuilt.additionalQuery.join(' OR '))}`
|
||
}
|
||
|
||
const payload = {
|
||
"status": "started",
|
||
"jobid": jobid,
|
||
"cost": cost,
|
||
"query": finalQuery,
|
||
"additionalQuery": queryBuilt.additionalQuery,
|
||
"doAdditionalQuery": queryBuilt.doAdditionalQuery,
|
||
"exportCount": body.exportCount,
|
||
"success": true,
|
||
"walletId": body.walletId
|
||
}
|
||
|
||
// Return to client that the job has been started
|
||
res.json(payload)
|
||
|
||
// End the response
|
||
res.end()
|
||
|
||
|
||
// Deduct clients from wallet
|
||
await removeWalletBalance(body.walletId, cost)
|
||
|
||
// Post to wallet DB
|
||
await axios.post(getRandServer().replace('BigData', 'Exports').replace('select', 'update'), {
|
||
"add": {
|
||
"doc": {
|
||
// uuid v4
|
||
"id": payload.jobid,
|
||
"cost": payload.cost,
|
||
"wallet": payload.walletId,
|
||
"query": payload.query,
|
||
"status": payload.status,
|
||
'count': payload.exportCount,
|
||
}
|
||
}
|
||
}, {
|
||
params: {
|
||
commit: true
|
||
}
|
||
})
|
||
|
||
// Base64 encode payload
|
||
const encodedPayload = Buffer.from(JSON.stringify(payload)).toString('base64')
|
||
|
||
// asyncronously run python3 exports/doExport.py <encodedPayload>
|
||
const pythonProcess = spawn('python3', ['exports/doExport.py', encodedPayload])
|
||
|
||
// Listen for data from the python process
|
||
pythonProcess.stdout.on('data', (data) => {
|
||
console.log(`Export Job ${jobid}: ${data}`);
|
||
})
|
||
|
||
// Listen for errors from the python process
|
||
pythonProcess.stderr.on('data', (data) => {
|
||
console.error(`Export Job ${jobid}: ${data}`);
|
||
})
|
||
} catch (e) {
|
||
console.error(e)
|
||
res.status(400).send('stoooop')
|
||
}
|
||
})
|
||
|
||
app.post('/exports/callbacks/a-unique-id/exportCb', async (req, res) => {
|
||
// Read body
|
||
const body = req.body
|
||
|
||
|
||
if (body.status === 'failed') {
|
||
// Re-credit wallet
|
||
await addWalletBalance(body.walletId, body.cost)
|
||
}
|
||
|
||
// Post to db
|
||
await axios.post(getRandServer().replace('BigData', 'Exports').replace('select', 'update'), {
|
||
"add": {
|
||
"doc": {
|
||
// uuid v4
|
||
"id": body.jobid,
|
||
"cost": body.cost,
|
||
"wallet": body.walletId,
|
||
"query": body.query,
|
||
"status": body.status,
|
||
'count': body.exportCount,
|
||
"link": body.link
|
||
}
|
||
}
|
||
}, {
|
||
params: {
|
||
commit: true
|
||
}
|
||
})
|
||
|
||
return res.status(200)
|
||
})
|
||
|
||
|
||
app.get('/wallet/:walletId', async (req, res) => {
|
||
try {
|
||
const walletId = req.params.walletId
|
||
|
||
|
||
if (!uuidRegex.test(walletId)) {
|
||
console.log('bad wallet id regex')
|
||
return res.status(400).json({ "error": "bad wallet id" })
|
||
}
|
||
|
||
if (typeof walletId !== 'string') {
|
||
return res.status(400).json({ "error": "bad request" })
|
||
}
|
||
|
||
if (walletId.length !== 36) {
|
||
console.log('bad wallet id length')
|
||
return res.status(400).json({ "error": "bad wallet id" })
|
||
}
|
||
|
||
let balance = await getWalletBalance(walletId)
|
||
|
||
return res.json({ "credits": balance })
|
||
} catch {
|
||
return res.status(400).json({ "error": "stooppp" })
|
||
}
|
||
})
|
||
|
||
|
||
function isAutomated(req: any) {
|
||
// Check if ip is in blacklist
|
||
const connectingIp = req.headers['cf-connecting-ip'] || req.connection.remoteAddress;
|
||
if (blacklistedAutomatedIps.includes(connectingIp)) {
|
||
console.log(`BLACKLISTED IP ${connectingIp}`)
|
||
return true
|
||
}
|
||
|
||
|
||
// Check if user agent has axios in it
|
||
const userAgent = req.headers['user-agent']
|
||
|
||
if (whitelistedUserAgents.some((whitelistedAgent) => userAgent.includes(whitelistedAgent))) {
|
||
return false
|
||
}
|
||
|
||
if (userAgent != undefined) {
|
||
if (typeof userAgent == 'string') {
|
||
if (Object.keys(req.query).length === 1 && req.query.emails) {
|
||
// Check if the user has gotten their wallet
|
||
const connectingIp = req.headers['cf-connecting-ip']
|
||
|
||
if (typeof connectingIp === 'string') {
|
||
if (hasRequestedWallet.includes(connectingIp)) {
|
||
return false
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
async function checkIPAutomatedSTDDEV(req: any) {
|
||
const ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress;
|
||
|
||
const userAgent = req.headers['user-agent']
|
||
|
||
if (whitelistedUserAgents.some((whitelistedAgent) => userAgent.includes(whitelistedAgent))) {
|
||
return false
|
||
}
|
||
|
||
if (ip === '0.0.0.0' || !ip) {
|
||
// Hello, KT :3c
|
||
console.log('wat r u doin')
|
||
return true
|
||
} else if (typeof ip === 'string') {
|
||
|
||
if (numLookupsPerIP[ip] === undefined) {
|
||
numLookupsPerIP[ip] = 0
|
||
}
|
||
|
||
// If no last query time
|
||
if (lastQueryTime[ip] === undefined) {
|
||
lastQueryTime[ip] = Date.now()
|
||
return false
|
||
} else {
|
||
// Get the time between the last query and now
|
||
const timeSinceLastQuery = (Date.now() - lastQueryTime[ip])
|
||
console.log(`Time since last query for ${ip}: ${timeSinceLastQuery}`)
|
||
|
||
numLookupsPerIP[ip] += 1
|
||
|
||
console.log(ip + ' has ' + numLookupsPerIP[ip] + ' lookups.')
|
||
|
||
if (numLookupsPerIP[ip] > 200) {
|
||
blacklistedAutomatedIps.push(ip)
|
||
return true
|
||
}
|
||
|
||
|
||
// Add current request to lookup table
|
||
if (lastQueryTimes[ip] === undefined) {
|
||
lastQueryTimes[ip] = []
|
||
}
|
||
|
||
// If there's less than 40 values in the array, add the current time
|
||
if (lastQueryTimes[ip].length < 40) {
|
||
lastQueryTimes[ip].push(timeSinceLastQuery)
|
||
}
|
||
// If there's more than 40 values in the array, remove the first value and add the current time
|
||
else if (lastQueryTimes[ip].length >= 40) {
|
||
lastQueryTimes[ip].shift()
|
||
lastQueryTimes[ip].push(timeSinceLastQuery)
|
||
}
|
||
|
||
// If there's more than 20 values in the array, calculate the standard deviation between the values
|
||
if (lastQueryTimes[ip].length >= 20) {
|
||
const standardDeviation = ss.standardDeviation(lastQueryTimes[ip])
|
||
console.log(`Standard deviation for ${ip}: ${standardDeviation}`)
|
||
// If the standard deviation is less than 1000, the user is probably a bot
|
||
if (standardDeviation < 1800) {
|
||
console.log('erm haht the duence')
|
||
blacklistedAutomatedIps.push(ip)
|
||
return true
|
||
}
|
||
}
|
||
|
||
// If all of the requests have less than 2000 ms of delay:
|
||
if (lastQueryTimes[ip].length >= 20) {
|
||
if (lastQueryTimes[ip].every((request) => request < 5000)) {
|
||
blacklistedAutomatedIps.push(ip)
|
||
return true
|
||
}
|
||
}
|
||
|
||
|
||
lastQueryTime[ip] = Date.now()
|
||
return false;
|
||
}
|
||
|
||
} else {
|
||
console.log('erm haht the duence')
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
async function doAutomatedRes(res: any, json?: boolean) {
|
||
if (json) {
|
||
return res.json({
|
||
resultCount: 69420,
|
||
count: 69420,
|
||
records: [
|
||
{
|
||
id: uuidv4(),
|
||
firstName: '[[EXTREMELY LOUD INCORRECT BUZZER]]',
|
||
lastName: 'Automated scraping detected. Please contact miyakoyakota@riseup.com to be whitelisted.',
|
||
email: 'This data comes from https://search.illicit.services and it is free to use. Do not pay for access to this data.',
|
||
}
|
||
]
|
||
})
|
||
}
|
||
else {
|
||
|
||
const rendered = mustache.render(recordsListingTemplate, {
|
||
resultCount: 69420,
|
||
count: 69420,
|
||
records: [
|
||
{
|
||
id: uuidv4(),
|
||
firstName: '[[EXTREMELY LOUD INCORRECT BUZZER]]',
|
||
fields: [
|
||
"firstName: [[EXTREMELY LOUD INCORRECT BUZZER]]",
|
||
"lastName: Automated scraping detected. Please contact miyakoyakota@riseup.com to be whitelisted. If you would like to use this site as an API, you may add ?wt=json to return JSON.",
|
||
"email: This data comes from https://search.illicit.services and it is free to use. Do not pay for access to this data.",
|
||
]
|
||
}
|
||
]
|
||
})
|
||
|
||
return res.send(rendered)
|
||
}
|
||
|
||
|
||
}
|
||
|
||
app.get('/documents/by_id/:id', async (req, res) => {
|
||
if (isAutomated(req)) {
|
||
return doAutomatedRes(res, req.query.wt === 'json')
|
||
}
|
||
|
||
if (await checkIPAutomatedSTDDEV(req)) {
|
||
return doAutomatedRes(res, req.query.wt === 'json')
|
||
}
|
||
|
||
const records = await queryForDocs(`id:${req.params.id}`)
|
||
const record = records.records[0]
|
||
|
||
if (record === undefined) {
|
||
return res.status(404).send('No record found.')
|
||
}
|
||
|
||
// If wt=json
|
||
if (req.query.wt == 'json') {
|
||
// check moreLikeThis
|
||
|
||
if (req.query.moreLikeThis == 'true') {
|
||
const similarRecords = await getSimilarRecords(record).catch((err) => {
|
||
console.log(err)
|
||
return []
|
||
})
|
||
|
||
return res.json({
|
||
record: record,
|
||
related: similarRecords
|
||
})
|
||
} else {
|
||
return res.json({
|
||
record: record
|
||
})
|
||
}
|
||
}
|
||
|
||
const similarRecords = await getSimilarRecords(record).catch((err) => {
|
||
return []
|
||
})
|
||
|
||
|
||
|
||
const rendered = mustache.render(recordByIdTemplate, {
|
||
id: record.id,
|
||
record: Object.entries(_.omit(record, ['id', '_version_'])).map(([key, value]) => {
|
||
return {
|
||
key: key,
|
||
value: value
|
||
}
|
||
}),
|
||
related: similarRecords.map((record) => {
|
||
return {
|
||
id: record.id,
|
||
record: Object.entries(_.omit(record, ['id', '_version_'])).map(([key, value]) => {
|
||
return {
|
||
key: key,
|
||
value: value
|
||
}
|
||
})
|
||
}
|
||
})
|
||
})
|
||
|
||
return res.send(rendered)
|
||
})
|
||
|
||
function makeid(length: number) {
|
||
let result = '';
|
||
const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
|
||
const charactersLength = characters.length;
|
||
let counter = 0;
|
||
while (counter < length) {
|
||
result += characters.charAt(Math.floor(Math.random() * charactersLength));
|
||
counter += 1;
|
||
}
|
||
return result;
|
||
}
|
||
|
||
app.get('/spatial', async (req, res) => {
|
||
if (req.query.latLong != undefined && typeof req.query.latLong === 'string') {
|
||
// Validate latLong using regex
|
||
const latLongRegex = /^-?\d{1,3}(?:\.\d{1,20})?,-?\d{1,3}(?:\.\d{1,20})?$/;
|
||
|
||
// Validate d within 0.1 and 1000
|
||
if (typeof req.query.d !== 'string' && req.query.d !== undefined) {
|
||
return res.send({
|
||
error: true,
|
||
errorMessage: 'Please provide a valid d between 0.1 and 1000'
|
||
})
|
||
}
|
||
const d = parseFloat(req.query.d || '0.2')
|
||
|
||
if (d < 0.1 || d > 1000) {
|
||
return res.send({
|
||
error: true,
|
||
errorMessage: 'Please provide a valid d between 0.1 and 1000'
|
||
})
|
||
}
|
||
|
||
if (latLongRegex.test(req.query.latLong)) {
|
||
let recordsResponse = await queryForDocsSpatial(req.query.latLong, d).catch((err) => {
|
||
return {
|
||
numDocs: 0,
|
||
records: [] as SolrRecord[]
|
||
}
|
||
})
|
||
|
||
res.json(recordsResponse)
|
||
|
||
} else {
|
||
// latLong is not valid
|
||
return res.send({
|
||
error: true,
|
||
errorMessage: 'Please only send coordinates as lat,long'
|
||
})
|
||
}
|
||
} else {
|
||
res.json({
|
||
error: true,
|
||
errorMessage: "Provide a latLong! Dumbo."
|
||
})
|
||
}
|
||
})
|
||
|
||
app.get('/records', async (req, res) => {
|
||
try {
|
||
const userAgent = req.headers['user-agent']
|
||
|
||
if (!(userAgent === 'yeayeyayaeyayeayeyaeyeayyaeyeyaeyae')) {
|
||
if (isAutomated(req)) {
|
||
return doAutomatedRes(res, req.query.wt === 'json')
|
||
}
|
||
}
|
||
|
||
if (req.query.wt === 'json') {
|
||
// Check if the user-agent is in the list of allowed user-agents
|
||
const apiKey = req.query.apikey || req.headers['user-agent']
|
||
|
||
if (typeof apiKey === 'string') {
|
||
if (whitelistedUserAgents.includes(apiKey)) {
|
||
// User agent is allowed
|
||
// Continue
|
||
} else {
|
||
console.log(`IP ${req.headers['cf-connecting-ip'] || req.connection.remoteAddress} tried to access the API with user-agent ${apiKey}`)
|
||
// User agent is not allowed
|
||
res.status(403)
|
||
return res.json({
|
||
error: true,
|
||
message: 'API Access requires a free API token. Please contact miyakoyakota@riseup.com to get one.'
|
||
})
|
||
}
|
||
} else {
|
||
return res.json({ error: true, message: 'API Access requires a free API token. Please contact miyakoyakota@riseup.com to get one.' })
|
||
}
|
||
}
|
||
|
||
let requestedQuery: Record<string, any> = {}
|
||
|
||
for (const [key, value] of Object.entries(req.query)) {
|
||
if (key !== 'wt') {
|
||
if (typeof value === 'string' || Array.isArray(value)) {
|
||
requestedQuery[key] = value
|
||
} else {
|
||
console.log(`Invalid query: ${key} is not a string or array. ${value}`)
|
||
}
|
||
}
|
||
}
|
||
|
||
const queryBuilt = buildQuery(requestedQuery, res as any) as any
|
||
|
||
if (!Boolean(queryBuilt.query)) {
|
||
return res.status(400).send('no query!!!')
|
||
}
|
||
|
||
// Check if type is express.Response
|
||
if (queryBuilt === undefined) {
|
||
return res.status(400).send('no query!!!')
|
||
}
|
||
|
||
if (!queryBuilt.query || !queryBuilt.additionalQuery || typeof queryBuilt.doAdditionalQuery !== 'boolean') {
|
||
return res.status(400).send('no query!!!')
|
||
}
|
||
|
||
let totalRecordCount = 0
|
||
let recordsResponse: any = null
|
||
|
||
recordsResponse = await queryForDocs(queryBuilt.query).catch((err) => {
|
||
console.log(err)
|
||
return {
|
||
numDocs: 0,
|
||
records: [] as SolrRecord[]
|
||
}
|
||
})
|
||
|
||
totalRecordCount += recordsResponse.numDocs
|
||
|
||
let records = recordsResponse.records
|
||
|
||
if (queryBuilt.doAdditionalQuery && !req.query.exact && req.query.exact != 'true' && req.query.exact != '1' && req.query.exact != 'yes' && req.query.exact != 'y' && req.query.exact != 'on' && req.query.exact != 't' && req.query.exact != 'sure' && req.query.exact != 'please' && req.query.exact != 'True') {
|
||
const additionalQueryBuilt = sanitizeQuery(queryBuilt.additionalQuery.join(' OR '))
|
||
|
||
let additionalRecordsResponse: any = null
|
||
|
||
if (typeof req.query.sofreshandsoclean === 'string') {
|
||
additionalRecordsResponse = await queryForDocs(additionalQueryBuilt, 10000).catch((err) => {
|
||
return {
|
||
numDocs: 0,
|
||
records: [] as SolrRecord[]
|
||
}
|
||
})
|
||
} else {
|
||
additionalRecordsResponse = await queryForDocs(additionalQueryBuilt).catch((err) => {
|
||
return {
|
||
numDocs: 0,
|
||
records: [] as SolrRecord[]
|
||
}
|
||
})
|
||
}
|
||
|
||
totalRecordCount += additionalRecordsResponse.numDocs
|
||
|
||
const additionalRecords = additionalRecordsResponse.records
|
||
|
||
records.push(...additionalRecords)
|
||
}
|
||
|
||
let recordsFinal = records.map((record: any) => {
|
||
return {
|
||
...record,
|
||
canMap: Boolean(record.address) || Boolean(record.latLong),
|
||
fields: Object.keys(_.omit(record, ['id', '_version_'])).map((key) => {
|
||
return `${key}: ${(record as any)[key]}`
|
||
})
|
||
}
|
||
})
|
||
|
||
// Remove duplicates
|
||
recordsFinal = _.uniqBy(recordsFinal, 'id')
|
||
|
||
// If wt=json is set return as json
|
||
if (req.query.wt == 'json') {
|
||
return res.json({
|
||
resultCount: totalRecordCount,
|
||
count: records.length,
|
||
records: recordsFinal
|
||
})
|
||
} else {
|
||
const rendered = mustache.render(recordsListingTemplate, {
|
||
resultCount: totalRecordCount,
|
||
count: records.length,
|
||
records: recordsFinal,
|
||
finalquery: queryBuilt.query
|
||
})
|
||
|
||
return res.send(rendered)
|
||
}
|
||
} catch (e) {
|
||
res.status(500)
|
||
}
|
||
})
|
||
|
||
app.listen(port, '0.0.0.0', () => {
|
||
console.log(`Example app listening on port ${port}`)
|
||
})
|