diff --git a/.travis.yml b/.travis.yml index 158ca137..8b252f2a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ dist: focal language: node_js node_js: - - 19.5.0 + - 19.6.0 notifications: diff --git a/.vscode/launch.json b/.vscode/launch.json index 9e89f4a8..9e3fe8b9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,10 +9,13 @@ "program": "${workspaceRoot}/boot.mjs", "cwd": "${workspaceRoot}", "env": { - "NODE_ENV": "development" + "NODE_ENV": "development", + "NODE_NO_WARNINGS":"1" }, "args": [ - "--no-deprecation" + "--no-deprecation", + "--loader ts-node/esm", + "--require ${workspaceRoot}/suppress-node-warnings.cjs", ], "skipFiles": [ "node_modules/**/*.js" diff --git a/boot.mjs b/boot.mjs index b159a36a..adafada6 100644 --- a/boot.mjs +++ b/boot.mjs @@ -3,63 +3,56 @@ import Fs from 'fs'; import Path from 'path'; import { exec } from 'child_process'; -import pjson from './package.json' assert { type: "json" }; -import * as GBServer from "./dist/src/app.js"; +import pjson from './package.json' assert { type: 'json' }; // Displays version of Node JS being used at runtime and others attributes. -console.log(`[GB Runtime] BotServer = v${pjson.version}`); -console.log(`[GB Runtime] BotLib = v${pjson.dependencies.botlib}`); -console.log(`[GB Runtime] BotBuilder (MS) = v${pjson.dependencies.botbuilder}`); -console.log(`[GB Runtime] NodeJS = ${process.version}`); -console.log(`[GB Runtime] platform = ${process.platform}`); -console.log(`[GB Runtime] architecture = ${process.arch}`); -console.log(`[GB Runtime] argv = ${process.argv}`); -console.log(`[GB Runtime] debugPort = ${process.debugPort}`); +console.log(``); +console.log(``); +console.log(``); +console.log(` █████ ██████ ███ ██ ██████ █████ █████ ██ █████ █████ ██████ ███ ® `); +console.log(` ██ ███ ██ █ ██ █ ██ ██ █ ██ ██ ███ ███ ██ ██████ █ * * █ ██ █ `); +console.log(` ██████ ██████ ██ ███ ██████ ██ ██ ██ ██ █████ █████ █████ ██ ████ 3.0`); +console.log(``); +console.debug(`botserver@${pjson.version}, botlib@${pjson.dependencies.botlib}, botbuilder@${pjson.dependencies.botbuilder}, nodeJS: ${process.version}, platform: ${process.platform}, architecture: ${process.arch}.`); var now = () => { - return (new Date()).toISOString().replace(/T/, ' ').replace(/\..+/, '') + ' UTC'; -} + return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '') + ' UTC'; +}; var __dirname = process.env.PWD; try { - - var run = () => { - - console.log(`[GB Runtime] Initializing General Bots (BotServer)...`); - console.log(`[GB Runtime] ${now()} - Running on '${import.meta.url}'`); - GBServer.GBServer.run(); - } - var processDist = () => { - if (!Fs.existsSync('dist')) { - console.log(`${now()} - Compiling...`); - exec(Path.join(__dirname, 'node_modules/.bin/tsc'), (err, stdout, stderr) => { - if (err) { - console.error(err); - return; - } - run(); - }); + var run = () => { + import('./dist/src/app.js').then((gb)=> gb.GBServer.run()); + }; + var processDist = () => { + if (!Fs.existsSync('dist')) { + console.log(`${now()} - Compiling...`); + exec(Path.join(__dirname, 'node_modules/.bin/tsc'), (err, stdout, stderr) => { + if (err) { + console.error(err); + return; } - else { - run(); - } - }; - - // Installing modules if it has not been done yet. - - if (!Fs.existsSync('node_modules')) { - console.log(`${now()} - Installing modules for the first time, please wait...`); - exec('npm install', (err, stdout, stderr) => { - if (err) { - console.error(err); - return; - } - processDist(); - }); - } - else { - processDist(); + run(); + }); + } else { + run(); } + }; + + // Installing modules if it has not been done yet. + + if (!Fs.existsSync('node_modules')) { + console.log(`${now()} - Installing modules for the first time, please wait...`); + exec('npm install', (err, stdout, stderr) => { + if (err) { + console.error(err); + return; + } + processDist(); + }); + } else { + processDist(); + } } catch (e) { - console.log(e); + console.log(e); } diff --git a/gbot.sh b/gbot.sh index 5dc1e7ea..3544eef4 100755 --- a/gbot.sh +++ b/gbot.sh @@ -1,5 +1,2 @@ -echo General Bots -echo Installing modules for the first time... - -npm i -node . +echo Starting General Bots... +npm run start diff --git a/package.json b/package.json index 65aca277..5081e869 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "Dário Vieira " ], "engines": { - "node": "=19.5.0" + "node": "=19.6.0" }, "license": "AGPL-3.0", "preferGlobal": true, @@ -34,7 +34,7 @@ "build-gbui": "cd packages/default.gbui && echo SKIP_PREFLIGHT_CHECK=true >.env && npm install && npm run build", "build-docs": "typedoc --options typedoc.json src/", "test": "node test.js", - "start": "node ./boot.cjs", + "start": "NODE_NO_WARNINGS=1 node ./boot.mjs --loader ts-node/esm --require ./suppress-node-warnings.cjs ", "reverse-proxy": "node_modules/.bin/ngrok http 4242", "watch:build": "tsc --watch", "posttypedoc": "shx cp .nojekyll docs/reference/.nojekyll", @@ -75,7 +75,8 @@ "adm-zip": "0.5.9", "alasql": "2.1.6", "any-shell-escape": "0.1.1", - "arraybuffer-to-buffer": "^0.0.7", + "arraybuffer-to-buffer": "0.0.7", + "async-mutex": "0.4.0", "async-promises": "0.2.3", "basic-auth": "2.0.1", "billboard.js": "3.6.3", @@ -86,7 +87,7 @@ "botbuilder-ai": "4.18.0", "botbuilder-dialogs": "4.18.0", "botframework-connector": "4.18.0", - "botlib": "3.0.2", + "botlib": "3.0.5", "c3-chart-maker": "0.2.8", "chatgpt": "2.4.2", "chrome-remote-interface": "0.31.3", @@ -106,6 +107,7 @@ "google-libphonenumber": "3.2.31", "googleapis": "109.0.1", "ibm-watson": "7.1.2", + "join-images-updated": "1.1.4", "keyv": "4.5.2", "koa": "2.13.4", "koa-body": "6.0.1", @@ -117,22 +119,22 @@ "ms-rest-azure": "3.0.0", "nexmo": "2.9.1", "node-cron": "3.0.2", + "node-html-parser": "6.1.5", "node-nlp": "4.24.0", "node-tesseract-ocr": "2.2.1", - "npm": "9.1.2", "open": "8.4.0", - "open-docxtemplater-image-module": "^1.0.3", + "open-docxtemplater-image-module": "1.0.3", "pdf-extraction": "1.0.2", - "pdf-to-png-converter": "^2.7.1", + "pdf-to-png-converter": "2.7.1", "pdfkit": "0.13.0", "phone": "3.1.30", "pizzip": "3.1.3", "pptxtemplater": "1.0.5", - "pragmatismo-io-framework": "1.0.20", + "pragmatismo-io-framework": "1.1.0", "prism-media": "1.3.4", "public-ip": "6.0.1", "punycode": "2.1.1", - "puppeteer": "19.2.2", + "puppeteer": "19.6.3", "puppeteer-extra": "3.3.4", "puppeteer-extra-plugin-stealth": "2.11.1", "qrcode": "1.5.1", @@ -142,11 +144,11 @@ "rimraf": "3.0.2", "safe-buffer": "5.2.1", "scanf": "1.1.2", - "sequelize": "6.25.7", - "sequelize-cli": "6.5.2", + "sequelize": "6.28.2", + "sequelize-cli": "6.6.0", "sequelize-typescript": "2.1.5", - "sharp": "^0.31.3", - "simple-git": "3.15.0", + "sharp": "0.31.3", + "simple-git": "3.16.0", "speakingurl": "14.0.1", "ssr-for-bots": "1.0.1-c", "strict-password-generator": "1.1.2", @@ -155,7 +157,7 @@ "tedious": "15.1.2", "textract": "2.5.0", "twitter-api-v2": "1.12.9", - "typescript": "4.9.3", + "typescript": "4.9.5", "typescript-rest-rpc": "1.0.7", "url-join": "5.0.0", "vbscript-to-typescript": "1.0.8", @@ -164,7 +166,7 @@ "vm2-process": "2.1.1", "walk-promise": "0.2.0", "washyourmouthoutwithsoap": "1.0.2", - "whatsapp-web.js": "1.18.3", + "whatsapp-web.js": "github:pedroslopez/whatsapp-web.js#fix-buttons-list", "winston": "3.8.2", "winston-logs-display": "1.0.0", "yarn": "1.22.19" @@ -181,8 +183,8 @@ "prettier-standard": "15.0.1", "semantic-release": "17.2.4", "simple-commit-message": "4.0.13", - "super-strong-password-generator": "^2.0.2", - "super-strong-password-generator-es": "^2.0.2", + "super-strong-password-generator": "2.0.2", + "super-strong-password-generator-es": "2.0.2", "travis-deploy-once": "5.0.11", "ts-node": "10.9.1", "tslint": "6.1.3" diff --git a/packages/admin.gbapp/dialogs/AdminDialog.ts b/packages/admin.gbapp/dialogs/AdminDialog.ts index 66761df1..24d95a58 100644 --- a/packages/admin.gbapp/dialogs/AdminDialog.ts +++ b/packages/admin.gbapp/dialogs/AdminDialog.ts @@ -238,6 +238,24 @@ export class AdminDialog extends IGBDialog { ]) ); + + min.dialogs.add( + new WaterfallDialog('/logs', [ + async step => { + if (step.context.activity.channelId !== 'msteams' && process.env.ENABLE_AUTH) { + return await step.beginDialog('/auth'); + } else { + return await step.next(step.options); + } + }, + async step => { + const logs = await min.core['getLatestLogs'](); + await min.conversationalService.sendText(min, step, logs); + return await step.replaceDialog('/ask', { isReturning: true }); + } + ])); + + min.dialogs.add( new WaterfallDialog('/publish', [ async step => { diff --git a/packages/basic.gblib/index.ts b/packages/basic.gblib/index.ts index b78ba356..b4d3eded 100644 --- a/packages/basic.gblib/index.ts +++ b/packages/basic.gblib/index.ts @@ -60,7 +60,8 @@ export class GBBasicPackage implements IGBPackage { public async loadPackage (core: IGBCoreService, sequelize: Sequelize): Promise { core.sequelize.addModels([GuaribasSchedule]); - app.use(koaBody.koaBody({ multipart: true })); + //app.use(koaBody.koaBody({ multipart: true, })); + app.listen(1111); } @@ -97,5 +98,11 @@ export class GBBasicPackage implements IGBPackage { app.use(waRouter.routes()); app.use(dbgRouter.routes()); app.use(imgRouter.routes()); + app.use(async (ctx, next) => { + if(ctx['status'] === 404){ + ctx.status = 404 + ctx.body = {msg:'emmmmmmm, seems 404'}; + } + }); } } diff --git a/packages/basic.gblib/services/ChartServices.ts b/packages/basic.gblib/services/ChartServices.ts index c2a2280a..25d6f686 100644 --- a/packages/basic.gblib/services/ChartServices.ts +++ b/packages/basic.gblib/services/ChartServices.ts @@ -32,7 +32,7 @@ 'use strict'; -import { createBrowser } from '../../core.gbapp/services/GBSSR.js'; +import { GBSSR }from '../../core.gbapp/services/GBSSR.js'; export class ChartServices { /** @@ -41,7 +41,7 @@ export class ChartServices { * @param {string} path screenshot image full path with file name */ public static async screenshot (args, path) { - const browser = await createBrowser(null); + const browser = await GBSSR.createBrowser(null); const page = await browser.newPage(); // load billboard.js assets from CDN. diff --git a/packages/basic.gblib/services/DialogKeywords.ts b/packages/basic.gblib/services/DialogKeywords.ts index 9eb8c008..dca4fab6 100644 --- a/packages/basic.gblib/services/DialogKeywords.ts +++ b/packages/basic.gblib/services/DialogKeywords.ts @@ -58,6 +58,7 @@ import mammoth from 'mammoth'; import qrcode from 'qrcode'; import { json } from 'body-parser'; import { WebAutomationServices } from './WebAutomationServices.js'; +import urljoin from 'url-join'; /** * Default check interval for user replay @@ -562,7 +563,7 @@ export class DialogKeywords { * */ public async sendFile({ pid, filename, caption }) { - const mobile = await this.userMobile({pid}); + const mobile = await this.userMobile({ pid }); GBLog.info(`BASIC: SEND FILE (current: ${mobile},filename '${filename}'.`); return await this.internalSendFile({ pid, mobile, filename, caption }); } @@ -606,10 +607,8 @@ export class DialogKeywords { return names.indexOf(name) > -1; } - - private async setOption({pid, name, value}) - { - if (this.isUserSystemParam(name)){ + private async setOption({ pid, name, value }) { + if (this.isUserSystemParam(name)) { throw new Error(`Not possible to define ${name} as it is a reserved system param name.`); } const process = GBServer.globals.processes[pid]; @@ -620,9 +619,8 @@ export class DialogKeywords { return { min, user, params }; } - private async getOption({pid, name}) - { - if (this.isUserSystemParam(name)){ + private async getOption({ pid, name }) { + if (this.isUserSystemParam(name)) { throw new Error(`Not possible to retrieve ${name} system param.`); } const process = GBServer.globals.processes[pid]; @@ -638,7 +636,7 @@ export class DialogKeywords { * */ public async setMaxLines({ pid, count }) { - await this.setOption({pid, name: "maxLines", value: count}); + await this.setOption({ pid, name: 'maxLines', value: count }); } /** @@ -647,8 +645,8 @@ export class DialogKeywords { * @example SET PARAM name AS value * */ - public async setUserParam({ pid, name, value }) { - await this.setOption({pid, name, value}); + public async setUserParam({ pid, name, value }) { + await this.setOption({ pid, name, value }); } /** @@ -657,11 +655,10 @@ export class DialogKeywords { * @example GET PARAM name * */ - public async getUserParam({ pid, name }) { - await this.getOption({pid, name}); + public async getUserParam({ pid, name }) { + await this.getOption({ pid, name }); } - /** * Defines the maximum lines to scan in spreedsheets. * @@ -669,7 +666,7 @@ export class DialogKeywords { * */ public async setMaxColumns({ pid, count }) { - await this.setOption({pid, name: "setMaxColumns", value: count}); + await this.setOption({ pid, name: 'setMaxColumns', value: count }); } /** @@ -679,8 +676,8 @@ export class DialogKeywords { * */ public async setWholeWord({ pid, on }) { - const value = (on.trim() === "on"); - await this.setOption({pid, name: "wholeWord", value: value}); + const value = on.trim() === 'on'; + await this.setOption({ pid, name: 'wholeWord', value: value }); } /** @@ -691,7 +688,7 @@ export class DialogKeywords { */ public async setTheme({ pid, theme }) { const value = theme.trim(); - await this.setOption({pid, name: "theme", value: value}); + await this.setOption({ pid, name: 'theme', value: value }); } /** @@ -701,14 +698,14 @@ export class DialogKeywords { * */ public async setTranslatorOn({ pid, on }) { - const value = (on.trim() === "on"); - await this.setOption({pid, name: "translatorOn", value: value}); + const value = on.trim() === 'on'; + await this.setOption({ pid, name: 'translatorOn', value: value }); } /** * Returns the name of the user acquired by WhatsApp API. */ - public async userName({pid}) { + public async userName({ pid }) { let { min, user, params } = await DialogKeywords.getProcessInfo(pid); return user.userName; } @@ -716,7 +713,7 @@ export class DialogKeywords { /** * Returns current mobile number from user in conversation. */ - public async userMobile({pid}) { + public async userMobile({ pid }) { let { min, user, params } = await DialogKeywords.getProcessInfo(pid); return user.userSystemId; } @@ -732,7 +729,6 @@ export class DialogKeywords { // return await beginDialog('/menu'); } - /** * Performs the transfer of the conversation to a human agent. * @@ -744,7 +740,7 @@ export class DialogKeywords { // return await beginDialog('/t',{ to: to }); } - public static getFileByHandle (hash) { + public static getFileByHandle(hash) { return GBServer.globals.files[hash]; } @@ -804,11 +800,11 @@ export class DialogKeywords { // await CollectionUtil.asyncForEach(args, async arg => { // i++; // list.sections[0].rows.push({ title: arg, id: `button${i}` }); - // await this.talk(arg); + // await this.getTalk(arg); // }); // const button = new wpp.Buttons(Messages[locale].choices, choices, ' ', ' '); - // await this.talk(button); + // await this.getTalk(button); GBLog.info(`BASIC: HEAR with [${args.toString()}] (Asking for input).`); } else { @@ -822,7 +818,7 @@ export class DialogKeywords { setTimeout(resolve, ms); }); }; - min.cbMap[userId] = {}; + min.cbMap[userId] = {}; min.cbMap[userId]['promise'] = '!GBHEAR'; while (min.cbMap[userId].promise === '!GBHEAR') { @@ -831,7 +827,68 @@ export class DialogKeywords { const answer = min.cbMap[userId].promise; - if (kind === 'file') { + if (kind === 'sheet') { + + // Retrieves the .xlsx file associated with the HEAR var AS file.xlsx. + + let { baseUrl, client } = await GBDeployer.internalGetDriveClient(this.min); + const botId = min.instance.botId; + const path = urljoin(`${botId}.gbai`, `${botId}.gbdata`); + let url = `${baseUrl}/drive/root:/${path}:/children`; + + GBLog.info(`Loading HEAR AS .xlsx options from Sheet: ${url}`); + const res = await client.api(url).get(); + + // Finds .xlsx specified by arg. + + const document = res.value.filter(m => { + return m.name === arg; + }); + if (document === undefined || document.length === 0) { + GBLog.info(`${arg} not found on .gbdata folder, check the package.`); + return null; + } + + // Reads all rows to be used as menu items in HEAR validation. + + let sheets = await client.api(`${baseUrl}/drive/items/${document[0].id}/workbook/worksheets`).get(); + const results = await client + .api( + `${baseUrl}/drive/items/${document[0].id}/workbook/worksheets('${sheets.value[0].name}')/range(address='A1:A256')` + ) + .get(); + + // Builds an array of items found in sheet file. + + let index = 0; + let list = []; + for (; index < results.text.length; index++) { + if (results.text[index][0] !== '') { + list.push( results.text[index][0]); + } + else + { + break; + } + } + + // Search the answer in one of valid list items loaded from sheeet. + + result = null; + await CollectionUtil.asyncForEach(list, async item => { + if (GBConversationalService.kmpSearch(answer, item) != -1) { + result = item; + } + }); + + // In case of unmatch, asks the person to try again. + + if (result === null) { + await this.getTalk({ pid, text: `Escolha por favor um dos itens sugeridos.` }); + return await this.getHear({ pid, kind, arg }); + } + + } else if (kind === 'file') { GBLog.info(`BASIC (${min.botId}): Upload done for ${answer.filename}.`); const handle = WebAutomationServices.cyrb53(this.min.botId + answer.filename); GBServer.globals.files[handle] = answer; @@ -850,7 +907,7 @@ export class DialogKeywords { const value = extractEntity(answer); if (value === null) { - await this.talk({ pid, text: 'Por favor, digite um e-mail válido.' }); + await this.getTalk({ pid, text: 'Por favor, digite um e-mail válido.' }); return await this.getHear({ pid, kind, arg }); } @@ -863,7 +920,7 @@ export class DialogKeywords { const value = extractEntity(answer); if (value === null || value.length != 1) { - await this.talk({ pid, text: 'Por favor, digite um nome válido.' }); + await this.getTalk({ pid, text: 'Por favor, digite um nome válido.' }); return await this.getHear({ pid, kind, arg }); } @@ -876,7 +933,7 @@ export class DialogKeywords { const value = extractEntity(answer); if (value === null || value.length != 1) { - await this.talk({ pid, text: 'Por favor, digite um número válido.' }); + await this.getTalk({ pid, text: 'Por favor, digite um número válido.' }); return await this.getHear({ pid, kind, arg }); } @@ -891,7 +948,7 @@ export class DialogKeywords { const value = extractEntity(answer); if (value === null || value.length != 1) { - await this.talk({ pid, text: 'Por favor, digite uma data no formato 12/12/2020.' }); + await this.getTalk({ pid, text: 'Por favor, digite uma data no formato 12/12/2020.' }); return await this.getHear({ pid, kind, arg }); } @@ -904,7 +961,7 @@ export class DialogKeywords { const value = extractEntity(answer); if (value === null || value.length != 1) { - await this.talk({ pid, text: 'Por favor, digite um horário no formato hh:ss.' }); + await this.getTalk({ pid, text: 'Por favor, digite um horário no formato hh:ss.' }); return await this.getHear({ pid, kind, arg }); } @@ -923,7 +980,7 @@ export class DialogKeywords { const value = extractEntity(answer); if (value === null || value.length != 1) { - await this.talk({ pid, text: 'Por favor, digite um valor monetário.' }); + await this.getTalk({ pid, text: 'Por favor, digite um valor monetário.' }); return await this.getHear({ pid, kind, arg }); } @@ -935,12 +992,12 @@ export class DialogKeywords { phoneNumber = phone(answer, { country: 'BRA' })[0]; phoneNumber = phoneUtil.parse(phoneNumber); } catch (error) { - await this.talk({ pid, text: Messages[locale].validation_enter_valid_mobile }); + await this.getTalk({ pid, text: Messages[locale].validation_enter_valid_mobile }); return await this.getHear({ pid, kind, arg }); } if (!phoneUtil.isPossibleNumber(phoneNumber)) { - await this.talk({ pid, text: 'Por favor, digite um número de telefone válido.' }); + await this.getTalk({ pid, text: 'Por favor, digite um número de telefone válido.' }); return await this.getHear({ pid, kind, arg }); } @@ -960,7 +1017,7 @@ export class DialogKeywords { const value = extractEntity(answer); if (value === null || value.length != 1) { - await this.talk({ pid, text: 'Por favor, digite um CEP válido.' }); + await this.getTalk({ pid, text: 'Por favor, digite um CEP válido.' }); return await this.getHear({ pid, kind, arg }); } @@ -975,7 +1032,7 @@ export class DialogKeywords { }); if (result === null) { - await this.talk({ pid, text: `Escolha por favor um dos itens sugeridos.` }); + await this.getTalk({ pid, text: `Escolha por favor um dos itens sugeridos.` }); return await this.getHear({ pid, kind, arg }); } } else if (kind === 'language') { @@ -1007,7 +1064,7 @@ export class DialogKeywords { }); if (result === null) { - await this.talk({ pid, text: `Escolha por favor um dos itens sugeridos.` }); + await this.getTalk({ pid, text: `Escolha por favor um dos itens sugeridos.` }); return await this.getHear({ pid, kind, arg }); } } @@ -1064,9 +1121,10 @@ export class DialogKeywords { /** * Talks to the user by using the specified text. */ - public async talk({ pid, text }) { + public async getTalk({ pid, text }) { GBLog.info(`BASIC: TALK '${text}'.`); const { min, user } = await DialogKeywords.getProcessInfo(pid); + if (user) { // TODO: const translate = this.user ? this.user.basicOptions.translatorOn : false; diff --git a/packages/basic.gblib/services/GBVMService.ts b/packages/basic.gblib/services/GBVMService.ts index d0bbfa90..1a972bea 100644 --- a/packages/basic.gblib/services/GBVMService.ts +++ b/packages/basic.gblib/services/GBVMService.ts @@ -32,7 +32,7 @@ 'use strict'; -import { GBLog, GBMinInstance, GBService, IGBCoreService, GBDialogStep } from 'botlib'; +import { GBMinInstance, GBService, IGBCoreService, GBDialogStep } from 'botlib'; import * as Fs from 'fs'; import { GBServer } from '../../../src/app.js'; import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; @@ -47,12 +47,10 @@ import walkPromise from 'walk-promise'; import child_process from 'child_process'; import Path from 'path'; import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; -import pkg from 'swagger-client'; import { DialogKeywords } from './DialogKeywords.js'; import { KeywordsExpressions } from './KeywordsExpressions.js'; -const { Swagger } = pkg; - - +import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js'; +import { GuaribasUser } from '../../security.gbapp/models/index.js'; /** * @fileoverview Decision was to priorize security(isolation) and debugging, @@ -63,10 +61,8 @@ const { Swagger } = pkg; * Basic services for BASIC manipulation. */ export class GBVMService extends GBService { - private static DEBUGGER_PORT = 9222; - public async loadDialogPackage(folder: string, min: GBMinInstance, core: IGBCoreService, deployer: GBDeployer) { const files = await walkPromise(folder); @@ -78,41 +74,46 @@ export class GBVMService extends GBService { let filename: string = file.name; if (filename.endsWith('.docx')) { - const wordFile = filename; - const vbsFile = filename.substr(0, filename.indexOf('docx')) + 'vbs'; - const fullVbsFile = urlJoin(folder, vbsFile); - const docxStat = Fs.statSync(urlJoin(folder, wordFile)); - const interval = 3000; // If compiled is older 30 seconds, then recompile. - let writeVBS = true; - if (Fs.existsSync(fullVbsFile)) { - const vbsStat = Fs.statSync(fullVbsFile); - if (docxStat['mtimeMs'] < vbsStat['mtimeMs'] + interval) { - writeVBS = false; - } - } - filename = vbsFile; - let mainName = GBVMService.getMethodNameFromVBSFilename(filename); - min.scriptMap[filename] = mainName; + filename = await this.loadDialog(filename, folder, min); + } + }); + } - if (writeVBS) { - let text = await this.getTextFromWord(folder, wordFile); + public async loadDialog(filename: string, folder: string, min: GBMinInstance) { + const wordFile = filename; + const vbsFile = filename.substr(0, filename.indexOf('docx')) + 'vbs'; + const fullVbsFile = urlJoin(folder, vbsFile); + const docxStat = Fs.statSync(urlJoin(folder, wordFile)); + const interval = 3000; // If compiled is older 30 seconds, then recompile. + let writeVBS = true; + if (Fs.existsSync(fullVbsFile)) { + const vbsStat = Fs.statSync(fullVbsFile); + if (docxStat['mtimeMs'] < vbsStat['mtimeMs'] + interval) { + writeVBS = false; + } + } + filename = vbsFile; + let mainName = GBVMService.getMethodNameFromVBSFilename(filename); + min.scriptMap[filename] = mainName; - const schedule = GBVMService.getSetScheduleKeywordArgs(text); - const s = new ScheduleServices(); - if (schedule) { - await s.createOrUpdateSchedule(min, schedule, mainName); - } else { - await s.deleteScheduleIfAny(min, mainName); - } - text = text.replace(/^\s*SET SCHEDULE (.*)/gim, ''); - Fs.writeFileSync(urlJoin(folder, vbsFile), text); - } + if (writeVBS) { + let text = await this.getTextFromWord(folder, wordFile); - // Process node_modules install. + const schedule = GBVMService.getSetScheduleKeywordArgs(text); + const s = new ScheduleServices(); + if (schedule) { + await s.createOrUpdateSchedule(min, schedule, mainName); + } else { + await s.deleteScheduleIfAny(min, mainName); + } + text = text.replace(/^\s*SET SCHEDULE (.*)/gim, ''); + Fs.writeFileSync(urlJoin(folder, vbsFile), text); + } - const node_modules = urlJoin(folder, 'node_modules'); - if (!Fs.existsSync(node_modules)) { - const packageJson = ` + // Process node_modules install. + const node_modules = urlJoin(folder, 'node_modules'); + if (!Fs.existsSync(node_modules)) { + const packageJson = ` { "name": "${min.botId}.gbdialog", "version": "1.0.0", @@ -127,43 +128,41 @@ export class GBVMService extends GBService { "vm2": "3.9.11" } }`; - Fs.writeFileSync(urlJoin(folder, 'package.json'), packageJson); + Fs.writeFileSync(urlJoin(folder, 'package.json'), packageJson); - GBLog.info(`BASIC: Installing .gbdialog node_modules for ${min.botId}...`); - const npmPath = urlJoin(process.env.PWD, 'node_modules', '.bin', 'npm'); - child_process.execSync(`${npmPath} install`, { cwd: folder }); - } + GBLogEx.info(min, `BASIC: Installing .gbdialog node_modules for ${min.botId}...`); + const npmPath = urlJoin(process.env.PWD, 'node_modules', '.bin', 'npm'); + child_process.execSync(`${npmPath} install`, { cwd: folder }); + } - // Hot swap for .vbs files. - - const fullFilename = urlJoin(folder, filename); - if (process.env.GBDIALOG_HOTSWAP) { - Fs.watchFile(fullFilename, async () => { - await this.translateBASIC(fullFilename, mainName, min.botId); - const parsedCode: string = Fs.readFileSync(jsfile, 'utf8'); - min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode; - }); - } - - const compiledAt = Fs.statSync(fullFilename); - const jsfile = urlJoin(folder, `${filename}.js`); - - if (Fs.existsSync(jsfile)) { - const jsStat = Fs.statSync(jsfile); - const interval = 30000; // If compiled is older 30 seconds, then recompile. - if (compiledAt.isFile() && compiledAt['mtimeMs'] > jsStat['mtimeMs'] + interval) { - await this.translateBASIC(fullFilename, mainName, min.botId); - } - } else { - await this.translateBASIC(fullFilename, mainName, min.botId); - } + // Hot swap for .vbs files. + const fullFilename = urlJoin(folder, filename); + if (process.env.GBDIALOG_HOTSWAP) { + Fs.watchFile(fullFilename, async () => { + await this.translateBASIC(fullFilename, mainName, min); const parsedCode: string = Fs.readFileSync(jsfile, 'utf8'); min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode; + }); + } + + const compiledAt = Fs.statSync(fullFilename); + const jsfile = urlJoin(folder, `${filename}.js`); + + if (Fs.existsSync(jsfile)) { + const jsStat = Fs.statSync(jsfile); + const interval = 30000; // If compiled is older 30 seconds, then recompile. + if (compiledAt.isFile() && compiledAt['mtimeMs'] > jsStat['mtimeMs'] + interval) { + await this.translateBASIC(fullFilename, mainName, min); } - }); + } else { + await this.translateBASIC(fullFilename, mainName, min); + } + const parsedCode: string = Fs.readFileSync(jsfile, 'utf8'); + min.sandBoxMap[mainName.toLowerCase().trim()] = parsedCode; + return filename; } - public async translateBASIC(filename: any, mainName: string, botId: string) { + public async translateBASIC(filename: any, mainName: string, min: GBMinInstance) { // Converts General Bots BASIC into regular VBS let basicCode: string = Fs.readFileSync(filename, 'utf8'); @@ -204,10 +203,10 @@ export class GBVMService extends GBService { // Interprocess communication from local HTTP to the BotServer. - const dk = rest.createClient('http://localhost:1111/api/v2/${botId}/dialog'); - const sys = rest.createClient('http://localhost:1111/api/v2/${botId}/system'); - const wa = rest.createClient('http://localhost:1111/api/v2/${botId}/webautomation'); - const img = rest.createClient('http://localhost:1111/api/v2/${botId}/imagprocessing'); + const dk = rest.createClient('http://localhost:1111/api/v2/${min.botId}/dialog'); + const sys = rest.createClient('http://localhost:1111/api/v2/${min.botId}/system'); + const wa = rest.createClient('http://localhost:1111/api/v2/${min.botId}/webautomation'); + const img = rest.createClient('http://localhost:1111/api/v2/${min.botId}/imagprocessing'); // Local variables. @@ -241,11 +240,13 @@ export class GBVMService extends GBService { ${code} + await wa.getCloseHandles({pid: pid}); + })(); `; Fs.writeFileSync(jsfile, code); - GBLog.info(`[GBVMService] Finished loading of ${filename}, JavaScript from Word: \n ${code}`); + GBLogEx.info(min, `[GBVMService] Finished loading of ${filename}, JavaScript from Word: \n ${code}`); } public static getMethodNameFromVBSFilename(filename: string) { @@ -262,22 +263,32 @@ export class GBVMService extends GBService { private async getTextFromWord(folder: string, filename: string) { return new Promise(async (resolve, reject) => { - textract.fromFileWithPath(urlJoin(folder, filename), { preserveLineBreaks: true }, (error, text) => { + const path = urlJoin(folder, filename); + textract.fromFileWithPath(path, { preserveLineBreaks: true }, (error, text) => { if (error) { - reject(error); - } else { - text = text.replace('¨', '"'); - text = text.replace('“', '"'); - text = text.replace('”', '"'); - text = text.replace('‘', "'"); - text = text.replace('’', "'"); - - resolve(text); + if (error.message.startsWith('File not correctly recognized as zip file')) { + text = Fs.readFileSync(path, 'utf8'); + } else { + reject(error); + } } + + if (text) { + text = GBVMService.normalizeQuotes(text); + } + resolve(text); }); }); } + public static normalizeQuotes(text: any) { + text = text.replace('¨', '"'); + text = text.replace('“', '"'); + text = text.replace('”', '"'); + text = text.replace('‘', "'"); + text = text.replace('’', "'"); + return text; + } /** * Converts General Bots BASIC @@ -288,7 +299,7 @@ export class GBVMService extends GBService { public async convert(code: string) { // Start and End of VB2TS tags of processing. - code = process.env.ENABLE_AUTH ? `hear gbLogin as login\n${code}` : code; + code = process.env.ENABLE_AUTH ? `hear GBLogExin as login\n${code}` : code; var lines = code.split('\n'); const keywords = KeywordsExpressions.getKeywords(); let current = 41; @@ -313,13 +324,18 @@ export class GBVMService extends GBService { /** * Executes the converted JavaScript from BASIC code inside execution context. */ - public static async callVM(text: string, min: GBMinInstance, step, deployer: GBDeployer, debug: boolean) { - + public static async callVM( + text: string, + min: GBMinInstance, + step, + user: GuaribasUser, + deployer: GBDeployer, + debug: boolean = false + ) { // Creates a class DialogKeywords which is the *this* pointer // in BASIC. - const user = step ? await min.userProfile.get(step.context, {}) : null; - const dk = new DialogKeywords(min, deployer, user); + const dk = new DialogKeywords(min, deployer, null); const sandbox = {}; const contentLocale = min.core.getParam( min.instance, @@ -327,17 +343,18 @@ export class GBVMService extends GBService { GBConfigService.get('DEFAULT_CONTENT_LANGUAGE') ); + // TODO: https://github.com/GeneralBots/BotServer/issues/217 // Auto-NLP generates BASIC variables related to entities. - if (step && step.context.activity['originalText']) { - const entities = await min['nerEngine'].findEntities(step.context.activity['originalText'], contentLocale); + // if (step && step.context.activity['originalText'] && min['nerEngine']) { + // const entities = await min['nerEngine'].findEntities(step.context.activity['originalText'], contentLocale); - for (let i = 0; i < entities.length; i++) { - const v = entities[i]; - const variableName = `${v.entity}`; - sandbox[variableName] = v.option; - } - } + // for (let i = 0; i < entities.length; i++) { + // const v = entities[i]; + // const variableName = `${v.entity}`; + // sandbox[variableName] = v.option; + // } + // } const botId = min.botId; const gbdialogPath = urlJoin(process.cwd(), 'work', `${botId}.gbai`, `${botId}.gbdialog`); @@ -353,9 +370,9 @@ export class GBVMService extends GBService { }; sandbox['id'] = dk.sys().getRandomId(); - sandbox['username'] = await dk.userName({pid}); - sandbox['mobile'] = await dk.userMobile({pid}); - sandbox['from'] = await dk.userMobile({pid}); + sandbox['username'] = await dk.userName({ pid }); + sandbox['mobile'] = await dk.userMobile({ pid }); + sandbox['from'] = await dk.userMobile({ pid }); sandbox['ENTER'] = String.fromCharCode(13); sandbox['headers'] = {}; sandbox['data'] = {}; @@ -364,8 +381,10 @@ export class GBVMService extends GBService { sandbox['httpPs'] = ''; sandbox['pid'] = pid; - if (GBConfigService.get('GBVM') === 'false') { - try { + let result; + + try { + if (GBConfigService.get('GBVM') === 'false') { const vm1 = new NodeVM({ allowAsync: true, sandbox: sandbox, @@ -379,23 +398,18 @@ export class GBVMService extends GBService { } }); const s = new VMScript(code, { filename: scriptPath }); - let x = vm1.run(s); - return x; - } catch (error) { - throw new Error(`BASIC RUNTIME ERR: ${error.message ? error.message : error}\n Stack:${error.stack}`); - } - } else { - const runnerPath = urlJoin( - process.cwd(), - 'dist', - 'packages', - 'basic.gblib', - 'services', - 'vm2-process', - 'vm2ProcessRunner.js' - ); + result = vm1.run(s); + } else { + const runnerPath = urlJoin( + process.cwd(), + 'dist', + 'packages', + 'basic.gblib', + 'services', + 'vm2-process', + 'vm2ProcessRunner.js' + ); - try { const { run } = createVm2Pool({ min: 0, max: 0, @@ -409,12 +423,13 @@ export class GBVMService extends GBService { script: runnerPath }); - const result = await run(code, { filename: scriptPath, sandbox: sandbox }); - - return result; - } catch (error) { - throw new Error(`BASIC RUNTIME ERR: ${error.message ? error.message : error}\n Stack:${error.stack}`); + result = await run(code, { filename: scriptPath, sandbox: sandbox }); } + } catch (error) { + throw new Error(`BASIC RUNTIME ERR: ${error.message ? error.message : error}\n Stack:${error.stack}`); + } finally { } + + return result; } } diff --git a/packages/basic.gblib/services/ImageProcessingServices.ts b/packages/basic.gblib/services/ImageProcessingServices.ts index 63dc84c8..9bd96ba0 100644 --- a/packages/basic.gblib/services/ImageProcessingServices.ts +++ b/packages/basic.gblib/services/ImageProcessingServices.ts @@ -32,9 +32,15 @@ 'use strict'; +import Path from 'path'; import { GBLog, GBMinInstance } from 'botlib'; import { DialogKeywords } from './DialogKeywords.js'; import sharp from 'sharp'; +import joinImages from 'join-images-updated'; +import { CollectionUtil } from 'pragmatismo-io-framework'; +import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; +import urlJoin from 'url-join'; +import { GBServer } from '../../../src/app.js'; /** * Image processing services of conversation to be called by BASIC. @@ -95,6 +101,32 @@ export class ImageProcessingServices { return; } + /** + * SET ORIENTATION VERTICAL + * + * file = MERGE file1, file2, file3 + */ + public async mergeImage({pid, files}) + { + const { min, user } = await DialogKeywords.getProcessInfo(pid); + + let paths = []; + await CollectionUtil.asyncForEach(files, async file => { + const gbfile = DialogKeywords.getFileByHandle(file); + paths.push(gbfile.path); + }); + + const botId = this.min.instance.botId; + const gbaiName = `${botId}.gbai`; + const img = await joinImages(paths); + const localName = Path.join('work', gbaiName, 'cache', `img-mrg${GBAdminService.getRndReadableIdentifier()}.png`); + const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName)); + img.toFile(localName); + + return { localName: localName, url: url, data: null }; + + } + /** * Sharpen the image. * diff --git a/packages/basic.gblib/services/KeywordsExpressions.ts b/packages/basic.gblib/services/KeywordsExpressions.ts index 26bf4f65..c9c7dafa 100644 --- a/packages/basic.gblib/services/KeywordsExpressions.ts +++ b/packages/basic.gblib/services/KeywordsExpressions.ts @@ -36,8 +36,7 @@ * Image processing services of conversation to be called by BASIC. */ export class KeywordsExpressions { - - private static getParams = (text: string, names) => { + private static getParams = (text: string, names) => { let ret = {}; const splitParamsButIgnoreCommasInDoublequotes = (str: string) => { return str.split(',').reduce( @@ -74,7 +73,6 @@ export class KeywordsExpressions { * Returns the list of BASIC keyword and their JS match. */ public static getKeywords() { - // Keywords from General Bots BASIC. let keywords = []; @@ -140,12 +138,28 @@ export class KeywordsExpressions { keywords[i++] = [ /^\s*open\s*(.*)/gim, ($0, $1, $2) => { + let sessionName; + let kind = ''; + let pos; + + if (pos = $1.match(/\s*AS\s*\#/)) { + kind = '"AS"'; + } else if (pos = $1.match(/\s*WITH\s*\#/)) { + kind = '"WITH"'; + } + + if (pos) { + let part = $1.substr($1.lastIndexOf(pos[0])); + sessionName = `"${part.substr(part.indexOf('#') + 1)}"`; + $1 = $1.substr(0, $1.lastIndexOf(pos[0])); + } + if (!$1.startsWith('"') && !$1.startsWith("'")) { $1 = `"${$1}"`; } const params = this.getParams($1, ['url', 'username', 'password']); - return `page = await wa.getPage({pid: pid,${params}})`; + return `page = await wa.getPage({pid: pid, sessionKind: ${kind}, sessionName: ${sessionName}, ${params}})`; } ]; @@ -156,6 +170,13 @@ export class KeywordsExpressions { } ]; + keywords[i++] = [ + /^\s*hear (\w+) as (\w+( \w+)*.xlsx)/gim, + ($0, $1, $2) => { + return `${$1} = await dk.getHear({pid: pid, kind:"sheet", arg: "${$2}"})`; + } + ]; + keywords[i++] = [ /^\s*hear (\w+) as\s*login/gim, ($0, $1) => { @@ -576,7 +597,7 @@ export class KeywordsExpressions { if ($3.substr(0, 1) !== '"') { $3 = `"${$3}"`; } - return `await dk.talk ({pid: pid, text: ${$3}})`; + return `await dk.getTalk ({pid: pid, text: ${$3}})`; } ]; @@ -677,6 +698,13 @@ export class KeywordsExpressions { } ]; + keywords[i++] = [ + /^\s*(MERGE)(\s*)(.*)/gim, + ($0, $1, $2, $3) => { + return `await img.mergeImage({pid: pid, files: [${$3}]})`; + } + ]; + keywords[i++] = [ /^\s*PRESS\s*(.*)/gim, ($0, $1, $2) => { diff --git a/packages/basic.gblib/services/ScheduleServices.ts b/packages/basic.gblib/services/ScheduleServices.ts index fc1ec471..e882cb51 100644 --- a/packages/basic.gblib/services/ScheduleServices.ts +++ b/packages/basic.gblib/services/ScheduleServices.ts @@ -139,7 +139,7 @@ export class ScheduleServices extends GBService { let min: GBMinInstance = GBServer.globals.minInstances.filter( p => p.instance.instanceId === item.instanceId )[0]; - await GBVMService.callVM(script, min, null, null, false); + await GBVMService.callVM(script, min, null, null, null, false); }; (async () => { await finalData(); diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index 0700e430..80a5d1e8 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -39,7 +39,7 @@ import { DialogKeywords } from './DialogKeywords.js'; import { GBServer } from '../../../src/app.js'; import { GBVMService } from './GBVMService.js'; import Fs from 'fs'; -import { createBrowser } from '../../core.gbapp/services/GBSSR.js'; +import { GBSSR }from '../../core.gbapp/services/GBSSR.js'; import urlJoin from 'url-join'; import Excel from 'exceljs'; import { TwitterApi } from 'twitter-api-v2'; @@ -93,11 +93,11 @@ export class SystemKeywords { } public async callVM({ pid, text }) { - const min = null; + const { min, user } = await DialogKeywords.getProcessInfo(pid); const step = null; const deployer = null; - return await GBVMService.callVM(text, min, step, deployer, false); + return await GBVMService.callVM(text, min, step, user, deployer, false); } public async append({ pid, args }) { @@ -259,7 +259,7 @@ export class SystemKeywords { const { min, user } = await DialogKeywords.getProcessInfo(pid); const gbaiName = `${min.botId}.gbai`; - const browser = await createBrowser(null); + const browser = await GBSSR.createBrowser(null); const page = await browser.newPage(); // Includes the associated CSS related to current theme. @@ -1462,6 +1462,7 @@ export class SystemKeywords { const images = []; let index = 0; path = Path.join(gbaiName, 'cache', `tmp${GBAdminService.getRndReadableIdentifier()}.docx`); + url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', Path.basename(localName)); const traverseDataToInjectImageUrl = async o => { for (var i in o) { diff --git a/packages/basic.gblib/services/WebAutomationServices.ts b/packages/basic.gblib/services/WebAutomationServices.ts index d2e873fb..c9aa7b66 100644 --- a/packages/basic.gblib/services/WebAutomationServices.ts +++ b/packages/basic.gblib/services/WebAutomationServices.ts @@ -32,19 +32,20 @@ 'use strict'; -import { GBLog, GBMinInstance } from 'botlib'; -import { GBServer } from '../../../src/app.js'; -import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; -import { createBrowser } from '../../core.gbapp/services/GBSSR.js'; -import { GuaribasUser } from '../../security.gbapp/models/index.js'; -import { DialogKeywords } from './DialogKeywords.js'; - -import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; import urlJoin from 'url-join'; import Fs from 'fs'; import Path from 'path'; import url from 'url'; -import { pid } from 'process'; + +import { GBLog, GBMinInstance } from 'botlib'; +import { GBServer } from '../../../src/app.js'; +import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js'; +import { GBSSR }from '../../core.gbapp/services/GBSSR.js'; +import { GuaribasUser } from '../../security.gbapp/models/index.js'; +import { DialogKeywords } from './DialogKeywords.js'; +import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js'; +import { Mutex } from 'async-mutex'; +import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js'; /** * Web Automation services of conversation to be called by BASIC. @@ -86,8 +87,6 @@ export class WebAutomationServices { */ maxLines: number = 2000; - pageMap = {}; - public static cyrb53 = (str, seed = 0) => { let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed; @@ -107,7 +106,7 @@ export class WebAutomationServices { * When creating this keyword facade,a bot instance is * specified among the deployer service. */ - constructor (min: GBMinInstance, user, dk) { + constructor(min: GBMinInstance, user, dk) { this.min = min; this.user = user; this.dk = dk; @@ -115,31 +114,106 @@ export class WebAutomationServices { this.debugWeb = this.min.core.getParam(this.min.instance, 'Debug Web Automation', false); } + public async getCloseHandles({ pid }) { + const { min, user } = await DialogKeywords.getProcessInfo(pid); + // Releases previous allocated OPEN semaphores. + + let keys = Object.keys(GBServer.globals.webSessions); + for (let i = 0; i < keys.length; i++) { + const session = GBServer.globals.webSessions[keys[i]]; + if (session.activePid === pid) { + session.semaphore.release(); + GBLogEx.info(min, `Release for PID: ${pid} done.`); + } + } + } + /** * Returns the page object. * * @example OPEN "https://wikipedia.org" */ - public async getPage ({ pid, url, username, password }) { - GBLog.info(`BASIC: Web Automation GET PAGE ${url}.`); - if (!this.browser) { - this.browser = await createBrowser(null); + + public async getPage({ pid, sessionKind, sessionName, url, username, password }) { + GBLog.info(`BASIC: Web Automation GET PAGE ${sessionName ? sessionName : ''} ${url}.`); + const { min, user } = await DialogKeywords.getProcessInfo(pid); + + let handle; + + // Try to find an existing handle. + + let session; + let keys = Object.keys(GBServer.globals.webSessions); + for (let i = 0; i < keys.length; i++) { + if (GBServer.globals.webSessions[keys[i]].sessionName === sessionName) { + session = GBServer.globals.webSessions[keys[i]]; + handle = keys[i]; + break; + } } - const page = (await this.browser.pages())[0]; - if (username || password) { - await page.authenticate({pid, username: username, password: password }); + + // Semaphore logic to block multiple entries on the same session. + + let page; + if (session) { + GBLogEx.info(min, `Acquiring (1) for PID: ${pid}...`); + const release = await session.semaphore.acquire(); + GBLogEx.info(min, `Acquire (1) for PID: ${pid} done.`); + try { + session.activePid = pid; + session.release = release; + page = session.page; + } catch { + release(); + } } + + // Creates the page if it is the first time. + + let browser; + if (!page) { + browser = await GBSSR.createBrowser(null); + page = (await browser.pages())[0]; + if (username || password) { + await page.authenticate({ pid, username: username, password: password }); + } + } + + // There is no session yet. + + if (!session && sessionKind === 'AS') { + + // A new web session is being created. + + handle = WebAutomationServices.cyrb53(this.min.botId + url); + GBServer.globals.webSessions[handle] = session = {}; + session.sessionName = sessionName; + + session.page = page; + session.browser = browser; + session.semaphore = new Mutex(); + GBLogEx.info(min, `Acquiring (2) for PID: ${pid}...`); + const release = await session.semaphore.acquire(); + GBLogEx.info(min, `Acquire (2) for PID: ${pid} done.`); + session.release = release; + session.activePid = pid; + + } + + // WITH is only valid in a previously defined session. + + if (!session && sessionKind == 'WITH') { + const error = `NULL session for OPEN WITH #${sessionName}.`; + GBLogEx.error(min, error); + } + await page.goto(url); - const handle = WebAutomationServices.cyrb53(this.min.botId + url); - - this.pageMap[handle] = page; - return handle; } - public getPageByHandle (hash) { - return this.pageMap[hash]; + public getPageByHandle(handle) { + return GBServer.globals.webSessions[handle].page; } /** @@ -147,7 +221,7 @@ export class WebAutomationServices { * * @example GET "selector" */ - public async getBySelector ({ handle, selector }) { + public async getBySelector({ handle, selector }) { const page = this.getPageByHandle(handle); GBLog.info(`BASIC: Web Automation GET element: ${selector}.`); await page.waitForSelector(selector); @@ -170,7 +244,7 @@ export class WebAutomationServices { * * @example GET page,"frameSelector,"elementSelector" */ - public async getByFrame ({ handle, frame, selector }) { + public async getByFrame({ handle, frame, selector }) { const page = this.getPageByHandle(handle); GBLog.info(`BASIC: Web Automation GET element by frame: ${selector}.`); await page.waitForSelector(frame); @@ -190,7 +264,7 @@ export class WebAutomationServices { /** * Simulates a mouse hover an web page element. */ - public async hover ({ pid, handle, selector }) { + public async hover({ pid, handle, selector }) { const page = this.getPageByHandle(handle); GBLog.info(`BASIC: Web Automation HOVER element: ${selector}.`); await this.getBySelector({ handle, selector: selector }); @@ -203,7 +277,7 @@ export class WebAutomationServices { * * @example CLICK page,"#idElement" */ - public async click ({ pid, handle, frameOrSelector, selector }) { + public async click({ pid, handle, frameOrSelector, selector }) { const page = this.getPageByHandle(handle); GBLog.info(`BASIC: Web Automation CLICK element: ${frameOrSelector}.`); if (selector) { @@ -219,7 +293,7 @@ export class WebAutomationServices { await this.debugStepWeb(pid, page); } - private async debugStepWeb (pid, page) { + private async debugStepWeb(pid, page) { let refresh = true; if (this.lastDebugWeb) { refresh = new Date().getTime() - this.lastDebugWeb.getTime() > 5000; @@ -229,7 +303,7 @@ export class WebAutomationServices { const mobile = this.min.core.getParam(this.min.instance, 'Bot Admin Number', null); const filename = page; if (mobile) { - await this.dk.sendFileTo({pid: pid, mobile, filename, caption: 'General Bots Debugger' }); + await this.dk.sendFileTo({ pid: pid, mobile, filename, caption: 'General Bots Debugger' }); } this.lastDebugWeb = new Date(); } @@ -240,7 +314,7 @@ export class WebAutomationServices { * * @example PRESS ENTER ON page */ - public async pressKey ({ handle, char, frame }) { + public async pressKey({ handle, char, frame }) { const page = this.getPageByHandle(handle); GBLog.info(`BASIC: Web Automation PRESS ${char} ON element: ${frame}.`); if (char.toLowerCase() === 'enter') { @@ -256,7 +330,7 @@ export class WebAutomationServices { } } - public async linkByText ({ pid, handle, text, index }) { + public async linkByText({ pid, handle, text, index }) { const page = this.getPageByHandle(handle); GBLog.info(`BASIC: Web Automation CLICK LINK TEXT: ${text} ${index}.`); if (!index) { @@ -272,7 +346,7 @@ export class WebAutomationServices { * * @example file = SCREENSHOT page */ - public async screenshot ({ handle, selector }) { + public async screenshot({ handle, selector }) { const page = this.getPageByHandle(handle); GBLog.info(`BASIC: Web Automation SCREENSHOT ${selector}.`); @@ -292,7 +366,7 @@ export class WebAutomationServices { * * @example SET page,"selector","text" */ - public async setElementText ({ pid, handle, selector, text }) { + public async setElementText({ pid, handle, selector, text }) { const page = this.getPageByHandle(handle); GBLog.info(`BASIC: Web Automation TYPE on ${selector}: ${text}.`); const e = await this.getBySelector({ handle, selector }); @@ -307,9 +381,9 @@ export class WebAutomationServices { * * @example file = DOWNLOAD element, folder */ - public async download ({ handle, selector, folder }) { + public async download({ handle, selector, folder }) { const page = this.getPageByHandle(handle); - + const element = await this.getBySelector({ handle, selector }); // https://github.com/GeneralBots/BotServer/issues/311 const container = element['_frame'] ? element['_frame'] : element['_page']; @@ -386,4 +460,20 @@ export class WebAutomationServices { return file; } + + private async recursiveFindInFrames (inputFrame, selector) { + const frames = inputFrame.childFrames(); + const results = await Promise.all( + frames.map(async frame => { + const el = await frame.$(selector); + if (el) return el; + if (frame.childFrames().length > 0) { + return await this.recursiveFindInFrames(frame, selector); + } + return null; + }) + ); + return results.find(Boolean); + } + } diff --git a/packages/core.gbapp/dialogs/WelcomeDialog.ts b/packages/core.gbapp/dialogs/WelcomeDialog.ts index 6f07b2ba..0ea22b06 100644 --- a/packages/core.gbapp/dialogs/WelcomeDialog.ts +++ b/packages/core.gbapp/dialogs/WelcomeDialog.ts @@ -72,16 +72,14 @@ export class WelcomeDialog extends IGBDialog { return step.replaceDialog(GBServer.globals.entryPointDialog); } - const user = await min.userProfile.get(step.context, {}); const locale = step.context.activity.locale; if ( - !user.once && + // TODO: https://github.com/GeneralBots/BotServer/issues/9 !user.once && step.context.activity.channelId === 'webchat' && min.core.getParam(min.instance, 'HelloGoodX', true) === 'true' ) { - user.once = true; - await min.userProfile.set(step.context, user); + // user.once = true; const a = new Date(); const date = a.getHours(); const msg = diff --git a/packages/core.gbapp/index.ts b/packages/core.gbapp/index.ts index f1a0a007..190a65fe 100644 --- a/packages/core.gbapp/index.ts +++ b/packages/core.gbapp/index.ts @@ -43,7 +43,7 @@ import { LanguageDialog } from './dialogs/LanguageDialog.js'; import { SwitchBotDialog } from './dialogs/SwitchBot.js'; import { WelcomeDialog } from './dialogs/WelcomeDialog.js'; import { WhoAmIDialog } from './dialogs/WhoAmIDialog.js'; -import { GuaribasChannel, GuaribasException, GuaribasInstance, GuaribasPackage } from './models/GBModel.js'; +import { GuaribasChannel, GuaribasInstance, GuaribasLog, GuaribasPackage } from './models/GBModel.js'; /** * Package for core.gbapp. @@ -53,7 +53,7 @@ export class GBCorePackage implements IGBPackage { public CurrentEngineName = 'guaribas-1.0.0'; public async loadPackage (core: IGBCoreService, sequelize: Sequelize): Promise { - core.sequelize.addModels([GuaribasInstance, GuaribasPackage, GuaribasChannel, GuaribasException]); + core.sequelize.addModels([GuaribasInstance, GuaribasPackage, GuaribasChannel, GuaribasLog]); } public async getDialogs (min: GBMinInstance) { diff --git a/packages/core.gbapp/models/GBModel.ts b/packages/core.gbapp/models/GBModel.ts index 203654db..76e3b17c 100644 --- a/packages/core.gbapp/models/GBModel.ts +++ b/packages/core.gbapp/models/GBModel.ts @@ -326,15 +326,18 @@ export class GuaribasChannel extends Model { */ @Table //tslint:disable-next-line:max-classes-per-file -export class GuaribasException extends Model { +export class GuaribasLog extends Model { @PrimaryKey @AutoIncrement @Column(DataType.INTEGER) - declare exceptionId: number; + declare logId: number; - @Column(DataType.STRING(255)) + @Column(DataType.STRING(1024)) declare message: string; + @Column(DataType.STRING(1)) + declare kind: string; + @ForeignKey(() => GuaribasInstance) @Column(DataType.INTEGER) declare instanceId: number; diff --git a/packages/core.gbapp/services/GBConversationalService.ts b/packages/core.gbapp/services/GBConversationalService.ts index 1786a256..a1f37005 100644 --- a/packages/core.gbapp/services/GBConversationalService.ts +++ b/packages/core.gbapp/services/GBConversationalService.ts @@ -581,14 +581,14 @@ export class GBConversationalService { await this.sendMarkdownToMobile(min, step, mobile, text); } else if (GBConfigService.get('DISABLE_WEB') !== 'true') { const html = marked(text); - await this.sendMarkdownToWeb(min, step, html, answer); + await this.sendHTMLToWeb(min, step, html, answer); } else { const html = marked(text); await min.conversationalService.sendText(min, step, html); } } - private async sendMarkdownToWeb (min, step: GBDialogStep, html: string, answer: string) { + private async sendHTMLToWeb (min, step: GBDialogStep, html: string, answer: string) { const locale = step.context.activity.locale; html = html.replace(/src\=\"kb\//gi, `src=\"../kb/`); diff --git a/packages/core.gbapp/services/GBCoreService.ts b/packages/core.gbapp/services/GBCoreService.ts index 2e52de62..9f387925 100644 --- a/packages/core.gbapp/services/GBCoreService.ts +++ b/packages/core.gbapp/services/GBCoreService.ts @@ -50,7 +50,7 @@ import { GBCustomerSatisfactionPackage } from '../../customer-satisfaction.gbapp import { GBKBPackage } from '../../kb.gbapp/index.js'; import { GBSecurityPackage } from '../../security.gbapp/index.js'; import { GBWhatsappPackage } from '../../whatsapp.gblib/index.js'; -import { GuaribasInstance } from '../models/GBModel.js'; +import { GuaribasInstance, GuaribasLog} from '../models/GBModel.js'; import { GBConfigService } from './GBConfigService.js'; import { GBAzureDeployerPackage } from '../../azuredeployer.gbapp/index.js'; import { GBSharePointPackage } from '../../sharepoint.gblib/index.js'; @@ -102,16 +102,16 @@ export class GBCoreService implements IGBCoreService { /** * */ - constructor () { + constructor() { this.adminService = new GBAdminService(this); } - public async ensureInstances (instances: IGBInstance[], bootInstance: any, core: IGBCoreService) {} + public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) {} /** * Gets database config and connect to storage. Currently two databases * are available: SQL Server and SQLite. */ - public async initStorage (): Promise { + public async initStorage(): Promise { this.dialect = GBConfigService.get('STORAGE_DIALECT'); let host: string | undefined; @@ -177,7 +177,7 @@ export class GBCoreService implements IGBCoreService { * Checks wheather storage is acessible or not and opens firewall * in case of any connection block. */ - public async checkStorage (installationDeployer: IGBInstallationDeployer) { + public async checkStorage(installationDeployer: IGBInstallationDeployer) { try { await this.sequelize.authenticate(); } catch (error) { @@ -195,7 +195,7 @@ export class GBCoreService implements IGBCoreService { /** * Syncronizes structure between model and tables in storage. */ - public async syncDatabaseStructure () { + public async syncDatabaseStructure() { if (GBConfigService.get('STORAGE_SYNC') === 'true') { const alter = GBConfigService.get('STORAGE_SYNC_ALTER') === 'true'; GBLog.info('Syncing database...'); @@ -213,7 +213,29 @@ export class GBCoreService implements IGBCoreService { /** * Loads all items to start several listeners. */ - public async loadInstances (): Promise { + public async getLatestLogs(instanceId: number): Promise { + const options = { + where: { + instanceId: instanceId, + state: 'active', + created: { + [Op.gt]: new Date(Date.now() - 60 * 60 * 1000 * 48) // Latest 48 hours. + } + } + }; + const list = await GuaribasLog.findAll(options); + let out = 'General Bots Log\n'; + await CollectionUtil.asyncForEach(list, async e => { + out = `${out}\n${e.createdAt} - ${e.message}`; + }); + return out; + } + + + /** + * Loads all items to start several listeners. + */ + public async loadInstances(): Promise { if (process.env.LOAD_ONLY !== undefined) { const bots = process.env.LOAD_ONLY.split(`;`); const and = []; @@ -236,7 +258,7 @@ export class GBCoreService implements IGBCoreService { /** * Loads just one Bot instance by its internal Id. */ - public async loadInstanceById (instanceId: number): Promise { + public async loadInstanceById(instanceId: number): Promise { const options = { where: { instanceId: instanceId, state: 'active' } }; return await GuaribasInstance.findOne(options); @@ -244,7 +266,7 @@ export class GBCoreService implements IGBCoreService { /** * Loads just one Bot instance. */ - public async loadInstanceByActivationCode (code: string): Promise { + public async loadInstanceByActivationCode(code: string): Promise { let options = { where: { activationCode: code, state: 'active' } }; return await GuaribasInstance.findOne(options); @@ -252,7 +274,7 @@ export class GBCoreService implements IGBCoreService { /** * Loads just one Bot instance. */ - public async loadInstanceByBotId (botId: string): Promise { + public async loadInstanceByBotId(botId: string): Promise { const options = { where: {} }; options.where = { botId: botId, state: 'active' }; @@ -264,7 +286,7 @@ export class GBCoreService implements IGBCoreService { * first startup, when user is asked some questions to create the * full base environment. */ - public async writeEnv (instance: IGBInstance) { + public async writeEnv(instance: IGBInstance) { const env = ` ADDITIONAL_DEPLOY_PATH= ADMIN_PASS=${instance.adminPass} @@ -294,7 +316,7 @@ ENDPOINT_UPDATE=true * when calling back from web services. This ensures that reverse proxy is * established. */ - public async ensureProxy (port): Promise { + public async ensureProxy(port): Promise { try { if (Fs.existsSync('node_modules/ngrok/bin/ngrok.exe') || Fs.existsSync('node_modules/ngrok/bin/ngrok')) { return await ngrok.connect({ port: port }); @@ -315,7 +337,7 @@ ENDPOINT_UPDATE=true * Setup generic web hooks so .gbapps can expose application logic * and get called on demand. */ - public installWebHook (isGet: boolean, url: string, callback: any) { + public installWebHook(isGet: boolean, url: string, callback: any) { if (isGet) { GBServer.globals.server.get(url, (req, res) => { callback(req, res); @@ -331,7 +353,7 @@ ENDPOINT_UPDATE=true * Defines the entry point dialog to be called whenever a user * starts talking to the bot. */ - public setEntryPointDialog (dialogName: string) { + public setEntryPointDialog(dialogName: string) { GBServer.globals.entryPointDialog = dialogName; } @@ -339,14 +361,14 @@ ENDPOINT_UPDATE=true * Replaces the default web application root path used to start the GB * with a custom home page. */ - public setWWWRoot (localPath: string) { + public setWWWRoot(localPath: string) { GBServer.globals.wwwroot = localPath; } /** * Removes a bot instance from storage. */ - public async deleteInstance (botId: string) { + public async deleteInstance(botId: string) { const options = { where: {} }; options.where = { botId: botId }; await GuaribasInstance.destroy(options); @@ -356,7 +378,7 @@ ENDPOINT_UPDATE=true * Saves a bot instance object to the storage handling * multi-column JSON based store 'params' field. */ - public async saveInstance (fullInstance: any) { + public async saveInstance(fullInstance: any) { const options = { where: {} }; options.where = { botId: fullInstance.botId }; let instance = await GuaribasInstance.findOne(options); @@ -377,7 +399,7 @@ ENDPOINT_UPDATE=true /** * Loads all bot instances from object storage, if it's formatted. */ - public async loadAllInstances ( + public async loadAllInstances( core: IGBCoreService, installationDeployer: IGBInstallationDeployer, proxyAddress: string @@ -431,7 +453,7 @@ ENDPOINT_UPDATE=true /** * Loads all system packages from 'packages' folder. */ - public async loadSysPackages (core: GBCoreService): Promise { + public async loadSysPackages(core: GBCoreService): Promise { // NOTE: if there is any code before this line a semicolon // will be necessary before this line. // Loads all system packages. @@ -469,7 +491,7 @@ ENDPOINT_UPDATE=true * Verifies that an complex global password has been specified * before starting the server. */ - public ensureAdminIsSecured () { + public ensureAdminIsSecured() { const password = GBConfigService.get('ADMIN_PASS'); if (!GBAdminService.StrongRegex.test(password)) { throw new Error( @@ -484,7 +506,7 @@ ENDPOINT_UPDATE=true * So a base main bot is always deployed and will act as root bot for * configuration tree with three levels: .env > root bot > all other bots. */ - public async createBootInstance ( + public async createBootInstance( core: GBCoreService, installationDeployer: IGBInstallationDeployer, proxyAddress: string @@ -519,7 +541,7 @@ ENDPOINT_UPDATE=true /** * Helper to get the web browser onpened in UI interfaces. */ - public openBrowserInDevelopment () { + public openBrowserInDevelopment() { if (process.env.NODE_ENV === 'development') { open('http://localhost:4242'); } @@ -540,35 +562,29 @@ ENDPOINT_UPDATE=true * // ' FOREIGN KEY ([groupId1], [groupId2]) REFERENCES [Group] ([groupId1], [groupId1]) ON DELETE NO ACTION,' + * // ' FOREIGN KEY ([instanceId]) REFERENCES [Instance] ([instanceId]) ON DELETE NO ACTION)' */ - private createTableQueryOverride (tableName, attributes, options): string { + private createTableQueryOverride(tableName, attributes, options): string { let sql: string = this.createTableQuery.apply(this.queryGenerator, [tableName, attributes, options]); const re1 = /CREATE\s+TABLE\s+\[([^\]]*)\]/; const matches = re1.exec(sql); if (matches !== null) { const table = matches[1]; const re2 = /PRIMARY\s+KEY\s+\(\[[^\]]*\](?:,\s*\[[^\]]*\])*\)/; - sql = sql.replace( - re2, - (match: string, ...args: any[]): string => { - return `CONSTRAINT [${table}_pk] ${match}`; - } - ); + sql = sql.replace(re2, (match: string, ...args: any[]): string => { + return `CONSTRAINT [${table}_pk] ${match}`; + }); const re3 = /FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g; const re4 = /\[([^\]]*)\]/g; - sql = sql.replace( - re3, - (match: string, ...args: any[]): string => { - const fkcols = args[0]; - let fkname = table; - let matches2 = re4.exec(fkcols); - while (matches2 !== null) { - fkname += `_${matches2[1]}`; - matches2 = re4.exec(fkcols); - } - - return `CONSTRAINT [${fkname}_fk] FOREIGN KEY (${fkcols})`; + sql = sql.replace(re3, (match: string, ...args: any[]): string => { + const fkcols = args[0]; + let fkname = table; + let matches2 = re4.exec(fkcols); + while (matches2 !== null) { + fkname += `_${matches2[1]}`; + matches2 = re4.exec(fkcols); } - ); + + return `CONSTRAINT [${fkname}_fk] FOREIGN KEY (${fkcols})`; + }); } return sql; @@ -582,7 +598,7 @@ ENDPOINT_UPDATE=true * ' CONSTRAINT [invalid2] FOREIGN KEY ([groupId1], [groupId2]) REFERENCES [Group] ([groupId1], [groupId2]) ON DELETE NO ACTION, ' + * ' CONSTRAINT [invalid3] FOREIGN KEY ([instanceId1]) REFERENCES [Instance] ([instanceId1]) ON DELETE NO ACTION' */ - private changeColumnQueryOverride (tableName, attributes): string { + private changeColumnQueryOverride(tableName, attributes): string { let sql: string = this.changeColumnQuery.apply(this.queryGenerator, [tableName, attributes]); const re1 = /ALTER\s+TABLE\s+\[([^\]]*)\]/; const matches = re1.exec(sql); @@ -590,20 +606,17 @@ ENDPOINT_UPDATE=true const table = matches[1]; const re2 = /(ADD\s+)?CONSTRAINT\s+\[([^\]]*)\]\s+FOREIGN\s+KEY\s+\((\[[^\]]*\](?:,\s*\[[^\]]*\])*)\)/g; const re3 = /\[([^\]]*)\]/g; - sql = sql.replace( - re2, - (match: string, ...args: any[]): string => { - const fkcols = args[2]; - let fkname = table; - let matches2 = re3.exec(fkcols); - while (matches2 !== null) { - fkname += `_${matches2[1]}`; - matches2 = re3.exec(fkcols); - } - - return `${args[0] ? args[0] : ''}CONSTRAINT [${fkname}_fk] FOREIGN KEY (${fkcols})`; + sql = sql.replace(re2, (match: string, ...args: any[]): string => { + const fkcols = args[2]; + let fkname = table; + let matches2 = re3.exec(fkcols); + while (matches2 !== null) { + fkname += `_${matches2[1]}`; + matches2 = re3.exec(fkcols); } - ); + + return `${args[0] ? args[0] : ''}CONSTRAINT [${fkname}_fk] FOREIGN KEY (${fkcols})`; + }); } return sql; @@ -612,7 +625,7 @@ ENDPOINT_UPDATE=true /** * Opens storage firewall used by the server when starting to get root bot instance. */ - private async openStorageFrontier (installationDeployer: IGBInstallationDeployer) { + private async openStorageFrontier(installationDeployer: IGBInstallationDeployer) { const group = GBConfigService.get('CLOUD_GROUP'); const serverName = GBConfigService.get('STORAGE_SERVER').split('.database.windows.net')[0]; await installationDeployer.openStorageFirewall(group, serverName); @@ -625,7 +638,7 @@ ENDPOINT_UPDATE=true * @param name Name of param to get from instance. * @param defaultValue Value returned when no param is defined in Config.xlsx. */ - public getParam (instance: IGBInstance, name: string, defaultValue?: T): any { + public getParam(instance: IGBInstance, name: string, defaultValue?: T): any { let value = null; if (instance.params) { const params = JSON.parse(instance.params); diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts index 087a3b80..5209c31c 100644 --- a/packages/core.gbapp/services/GBDeployer.ts +++ b/packages/core.gbapp/services/GBDeployer.ts @@ -223,7 +223,7 @@ export class GBDeployer implements IGBDeployer { instance.activationCode = instance.botId; instance.state = 'active'; instance.nlpScore = 0.8; - instance.searchScore = 0.45; + instance.searchScore = 0.25; instance.whatsappServiceKey = null; instance.whatsappServiceNumber = null; instance.whatsappServiceUrl = null; @@ -713,6 +713,7 @@ export class GBDeployer implements IGBDeployer { public async rebuildIndex(instance: IGBInstance, searchSchema: any) { // Prepares search. + const search = new AzureSearch( instance.searchKey, instance.searchHost, @@ -721,40 +722,28 @@ export class GBDeployer implements IGBDeployer { ); const connectionString = GBDeployer.getConnectionStringFromInstance(instance); const dsName = 'gb'; - - // Removes any previous index. - try { - await search.deleteDataSource(dsName); + await search.createDataSource(dsName, dsName, 'GuaribasQuestion', 'azuresql', connectionString); } catch (err) { - // If it is a 404 there is nothing to delete as it is the first creation. + GBLog.error(err); - if (err.code !== 404) { - throw err; - } } // Removes the index. try { - await search.deleteIndex(); + await search.createIndex(searchSchema, dsName); } catch (err) { // If it is a 404 there is nothing to delete as it is the first creation. if (err.code !== 404 && err.code !== 'OperationNotAllowed') { - throw err; + } } - // Creates the data source and index on the cloud. - try { - await search.createDataSource(dsName, dsName, 'GuaribasQuestion', 'azuresql', connectionString); - } catch (err) { - GBLog.error(err); - throw err; - } - await search.createIndex(searchSchema, dsName); + await search.rebuildIndex(instance.searchIndexer); + } /** diff --git a/packages/core.gbapp/services/GBLogEx.ts b/packages/core.gbapp/services/GBLogEx.ts new file mode 100644 index 00000000..a57db04f --- /dev/null +++ b/packages/core.gbapp/services/GBLogEx.ts @@ -0,0 +1,86 @@ +/*****************************************************************************\ +| ( )_ _ | +| _ _ _ __ _ _ __ ___ ___ _ _ | ,_)(_) ___ _ _ _ | +| ( '_`\ ( '__)/'_` ) /'_ `\/' _ ` _ `\ /'_` )| | | |/',__)/ \ /`\ /'_`\ | +| | (_) )| | ( (_| |( (_) || ( ) ( ) |( (_| || |_ | |\__, \| |*| |( (_) ) | +| | ,__/'(_) `\__,_)`\__ |(_) (_) (_)`\__,_)`\__)(_)(____/(_) (_)`\___/' | +| | | ( )_) | | +| (_) \___/' | +| | +| General Bots Copyright (c) Pragmatismo.io. All rights reserved. | +| Licensed under the AGPL-3.0. | +| | +| According to our dual licensing model, this program can be used either | +| under the terms of the GNU Affero General Public License, version 3, | +| or under a proprietary license. | +| | +| The texts of the GNU Affero General Public License with an additional | +| permission and of our proprietary license can be found at and | +| in the LICENSE file you have received along with this program. | +| | +| This program is distributed in the hope that it will be useful, | +| but WITHOUT ANY WARRANTY, without even the implied warranty of | +| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | +| GNU Affero General Public License for more details. | +| | +| "General Bots" is a registered trademark of Pragmatismo.io. | +| The licensing of the program under the AGPLv3 does not imply a | +| trademark license. Therefore any rights, title and interest in | +| our trademarks remain entirely with us. | +| | +\*****************************************************************************/ + +/** + * @fileoverview General Bots server core. + */ + +'use strict'; + +import { GBLog, IGBInstance } from 'botlib'; +import { GuaribasLog } from '../models/GBModel.js'; + +export class GBLogEx { + public static async error(minOrInstanceId: any, message: string) { + if (typeof minOrInstanceId === 'object') { + minOrInstanceId = minOrInstanceId.instance.instanceId; + } + GBLog.error(`${minOrInstanceId}: ${message}.`); + await this.log(minOrInstanceId, 'e', message); + } + + public static async debug(minOrInstanceId: any, message: string) { + if (typeof minOrInstanceId === 'object') { + minOrInstanceId = minOrInstanceId.instance.instanceId; + } + GBLog.debug(`${minOrInstanceId}: ${message}.`); + await this.log(minOrInstanceId, 'd', message); + } + + public static async info(minOrInstanceId: any, message: string) { + if (typeof minOrInstanceId === 'object') { + minOrInstanceId = minOrInstanceId.instance.instanceId; + } + GBLog.info(`${minOrInstanceId}: ${message}.`); + await this.log(minOrInstanceId, 'i', message); + } + + public static async verbose(minOrInstanceId: any, message: string) { + if (typeof minOrInstanceId === 'object') { + minOrInstanceId = minOrInstanceId.instance.instanceId; + } + GBLog.verbose(`${minOrInstanceId}: ${message}.`); + await this.log(minOrInstanceId, 'v', message); + } + + /** + * Finds and update user agent information to a next available person. + */ + public static async log(instance: IGBInstance, kind: string, message: string): Promise { + message = message ? message.substring(0, 1023) : null; + return await GuaribasLog.create({ + instanceId: instance ? instance.instanceId : 1, + message: message, + kind: kind + }); + } +} diff --git a/packages/core.gbapp/services/GBMinService.ts b/packages/core.gbapp/services/GBMinService.ts index 09f9c3d5..87b2074f 100644 --- a/packages/core.gbapp/services/GBMinService.ts +++ b/packages/core.gbapp/services/GBMinService.ts @@ -87,6 +87,7 @@ import { GoogleChatDirectLine } from '../../google-chat.gblib/services/GoogleCha import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js'; import * as nlp from 'node-nlp'; import Path from 'path'; +import { GBSSR } from './GBSSR.js'; /** * Minimal service layer for a bot and encapsulation of BOT Framework calls. @@ -95,7 +96,7 @@ export class GBMinService { /** * Default General Bots User Interface package. */ - private static uiPackage = 'default.gbui'; + public static uiPackage = 'default.gbui'; /** * Main core service attached to this bot service. @@ -141,23 +142,11 @@ export class GBMinService { // Servers default UI on root address '/' if web enabled. if (process.env.DISABLE_WEB !== 'true') { - // SSR processing. + // SSR processing and default.gbui access definition. - const defaultOptions = { - prerender: [], - exclude: ['/api/', '/instances/', '/webhooks/'], - useCache: true, - cacheRefreshRate: 86400 - }; - // GBServer.globals.server.use(ssrForBots(defaultOptions)); - - const url = GBServer.globals.wwwroot - ? GBServer.globals.wwwroot - : urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'); - - // default.gbui access definition. - - GBServer.globals.server.use('/', express.static(url)); + GBServer.globals.server.get('/', async (req, res, next) => { + await GBSSR.ssrFilter(req, res, next); + }); // Servers the bot information object via HTTP so clients can get // instance information stored on server. @@ -240,6 +229,39 @@ export class GBMinService { removeRoute(GBServer.globals.server, uiUrl); GBServer.globals.minInstances = GBServer.globals.minInstances.filter(p => p.instance.botId !== botId); + } + + /** + * Mount the bot web site (default.gbui) secure domain. + */ + public async loadDomain(min: GBMinInstance) { + // TODO: https://github.com/GeneralBots/BotServer/issues/321 + const options = { + passphrase: process.env.CERTIFICATE2_PASSPHRASE, + pfx: Fs.readFileSync(process.env.CERTIFICATE2_PFX) + }; + + const domain = min.core.getParam(min.instance, 'Domain', null); + if (domain) { + GBServer.globals.server.get(domain, async (req, res, next) => { + await GBSSR.ssrFilter(req, res, next); + }); + GBLog.verbose(`Bot UI ${GBMinService.uiPackage} accessible at custom domain: ${domain}.`); + } + + + GBServer.globals.httpsServer.addContext(process.env.CERTIFICATE2_DOMAIN, options); + } + + /** + * Unmounts the bot web site (default.gbui) secure domain, if any. + */ + public async unloadDomain(instance: IGBInstance) { + + + + + } /** @@ -295,6 +317,10 @@ export class GBMinService { if (!Fs.existsSync(dir)) { mkdirp.sync(dir); } + dir = `work/${min.botId}.gbai/${min.botId}.gbui`; + if (!Fs.existsSync(dir)) { + mkdirp.sync(dir); + } // Loads Named Entity data for this bot. @@ -370,23 +396,15 @@ export class GBMinService { if (process.env.DISABLE_WEB !== 'true') { const uiUrl = `/${instance.botId}`; + + GBServer.globals.server.get(uiUrl, async (req, res, next) => { + await GBSSR.ssrFilter(req, res, next); + }); const uiUrlAlt = `/${instance.activationCode}`; - GBServer.globals.server.use( - uiUrl, - express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build')) - ); - GBServer.globals.server.use( - uiUrlAlt, - express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build')) - ); - const domain = min.core.getParam(min.instance, 'Domain', null); - if (domain) { - GBServer.globals.server.use( - domain, - express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build')) - ); - GBLog.verbose(`Bot UI ${GBMinService.uiPackage} accessible at custom domain: ${domain}.`); - } + GBServer.globals.server.get(uiUrlAlt, async (req, res, next) => { + await GBSSR.ssrFilter(req, res, next); + }); + GBLog.verbose(`Bot UI ${GBMinService.uiPackage} accessible at: ${uiUrl} and ${uiUrlAlt}.`); } @@ -599,6 +617,7 @@ export class GBMinService { * Gets a Speech to Text / Text to Speech token from the provider. */ private async getSTSToken(instance: any) { + return null; // TODO: https://github.com/GeneralBots/BotServer/issues/332 const options = { method: 'POST', headers: { @@ -884,12 +903,7 @@ export class GBMinService { data: data.slice(0, 10) }); } - - // Saves session user (persisted GuaribasUser is inside). - - await min.userProfile.set(step.context, user); } - // Required for MSTEAMS handling of persisted conversations. if (step.context.activity.channelId === 'msteams') { @@ -927,7 +941,7 @@ export class GBMinService { if (startDialog) { await sec.setParam(userId, 'welcomed', 'true'); GBLog.info(`Auto start (teams) dialog is now being called: ${startDialog} for ${min.instance.botId}...`); - await GBVMService.callVM(startDialog.toLowerCase(), min, step, this.deployer, false); + await GBVMService.callVM(startDialog.toLowerCase(), min, step, user, this.deployer, false); } } } @@ -973,7 +987,7 @@ export class GBMinService { GBLog.info( `Auto start (web 1) dialog is now being called: ${startDialog} for ${min.instance.instanceId}...` ); - await GBVMService.callVM(startDialog.toLowerCase(), min, step, this.deployer, false); + await GBVMService.callVM(startDialog.toLowerCase(), min, step, user, this.deployer, false); } } } else { @@ -987,11 +1001,10 @@ export class GBMinService { ) { await sec.setParam(userId, 'welcomed', 'true'); min['conversationWelcomed'][step.context.activity.conversation.id] = true; - await min.userProfile.set(step.context, user); GBLog.info( `Auto start (whatsapp) dialog is now being called: ${startDialog} for ${min.instance.instanceId}...` ); - await GBVMService.callVM(startDialog.toLowerCase(), min, step, this.deployer, false); + await GBVMService.callVM(startDialog.toLowerCase(), min, step, user, this.deployer, false); } } } @@ -1004,10 +1017,6 @@ export class GBMinService { await this.processEventActivity(min, user, context, step); } - - // Saves conversation state for later use. - - await conversationState.saveChanges(context, true); } catch (error) { const msg = `ERROR: ${error.message} ${error.stack ? error.stack : ''}`; GBLog.error(msg); @@ -1051,7 +1060,7 @@ export class GBMinService { if (startDialog && !min['conversationWelcomed'][step.context.activity.conversation.id]) { user.welcomed = true; GBLog.info(`Auto start (web 2) dialog is now being called: ${startDialog} for ${min.instance.instanceId}...`); - await GBVMService.callVM(startDialog.toLowerCase(), min, step, this.deployer, false); + await GBVMService.callVM(startDialog.toLowerCase(), min, step, user, this.deployer, false); } } else if (context.activity.name === 'updateToken') { const token = context.activity.data; @@ -1133,6 +1142,7 @@ export class GBMinService { const member = context.activity.from; let user = await sec.ensureUser(min.instance.instanceId, member.id, member.name, '', 'web', member.name, null); + const userId = user.userId; const params = user.params ? JSON.parse(user.params) : {}; @@ -1155,6 +1165,9 @@ export class GBMinService { userId, context.activity.text ); + + const conversationReference = JSON.stringify(TurnContext.getConversationReference(context.activity)); + await sec.updateConversationReferenceById(userId, conversationReference); } } @@ -1202,7 +1215,7 @@ export class GBMinService { const isVMCall = Object.keys(min.scriptMap).find(key => min.scriptMap[key] === context.activity.text) !== undefined; if (isVMCall) { - await GBVMService.callVM(context.activity.text, min, step, this.deployer, false); + await GBVMService.callVM(context.activity.text, min, step, user, this.deployer, false); } else if (context.activity.text.charAt(0) === '/') { const text = context.activity.text; const parts = text.split(' '); @@ -1212,16 +1225,13 @@ export class GBMinService { if (cmdOrDialogName === '/start') { // Reset user. - const user = await min.userProfile.get(context, {}); await min.conversationalService.sendEvent(min, step, 'loadInstance', {}); - user.loaded = false; - await min.userProfile.set(step.context, user); } else if (cmdOrDialogName === '/call') { - await GBVMService.callVM(args, min, step, this.deployer, false); + await GBVMService.callVM(args, min, step, user, this.deployer, false); } else if (cmdOrDialogName === '/callsch') { - await GBVMService.callVM(args, min, null, null, false); + await GBVMService.callVM(args, min, null, null, null, false); } else if (cmdOrDialogName === '/calldbg') { - await GBVMService.callVM(args, min, step, this.deployer, true); + await GBVMService.callVM(args, min, step, user, this.deployer, true); } else { await step.beginDialog(cmdOrDialogName, { args: args }); } diff --git a/packages/core.gbapp/services/GBSSR.ts b/packages/core.gbapp/services/GBSSR.ts index 3148c4c7..cd12c242 100644 --- a/packages/core.gbapp/services/GBSSR.ts +++ b/packages/core.gbapp/services/GBSSR.ts @@ -36,290 +36,285 @@ 'use strict'; -import puppeteer from 'puppeteer-extra'; +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +import Path from 'path'; import Fs from 'fs'; - -// const StealthPlugin from 'puppeteer-extra-plugin-stealth') -// puppeteer.use(StealthPlugin()); - import { NextFunction, Request, Response } from 'express'; import urljoin from 'url-join'; +import { GBMinInstance } from 'botlib'; +import { GBServer } from '../../../src/app.js'; +import { GBLogEx } from './GBLogEx.js'; +import urlJoin from 'url-join'; +import { GBDeployer } from './GBDeployer.js'; +import { GBMinService } from './GBMinService.js'; +const puppeteer = require('puppeteer-extra'); +const hidden = require('puppeteer-extra-plugin-stealth'); +const { executablePath } = require('puppeteer'); -// https://hackernoon.com/tips-and-tricks-for-web-scraping-with-puppeteer-ed391a63d952 -// Dont download all resources, we just need the HTML -// Also, this is huge performance/response time boost -const blockedResourceTypes = ['image', 'media', 'font', 'texttrack', 'object', 'beacon', 'csp_report', 'imageset']; -// const whitelist = ["document", "script", "xhr", "fetch"]; -const skippedResources = [ - 'quantserve', - 'adzerk', - 'doubleclick', - 'adition', - 'exelator', - 'sharethrough', - 'cdn.api.twitter', - 'google-analytics', - 'googletagmanager', - 'google', - 'fontawesome', - 'facebook', - 'analytics', - 'optimizely', - 'clicktale', - 'mixpanel', - 'zedo', - 'clicksor', - 'tiqcdn' -]; - -const RENDER_CACHE = new Map(); - -async function createBrowser (profilePath): Promise { - let args = [ - '--check-for-update-interval=2592000', - '--disable-accelerated-2d-canvas', - '--disable-dev-shm-usage', - '--disable-features=site-per-process', - '--disable-gpu', - '--no-first-run', - '--no-default-browser-check' +export class GBSSR { + // https://hackernoon.com/tips-and-tricks-for-web-scraping-with-puppeteer-ed391a63d952 + // Dont download all resources, we just need the HTML + // Also, this is huge performance/response time boost + private static blockedResourceTypes = [ + 'image', + 'media', + 'font', + 'texttrack', + 'object', + 'beacon', + 'csp_report', + 'imageset' ]; - if (profilePath) { - args.push(`--user-data-dir=${profilePath}`); + // const whitelist = ["document", "script", "xhr", "fetch"]; + private static skippedResources = [ + 'quantserve', + 'adzerk', + 'doubleclick', + 'adition', + 'exelator', + 'sharethrough', + 'cdn.api.twitter', + 'google-analytics', + 'googletagmanager', + 'google', + 'fontawesome', + 'facebook', + 'analytics', + 'optimizely', + 'clicktale', + 'mixpanel', + 'zedo', + 'clicksor', + 'tiqcdn' + ]; - const preferences = urljoin(profilePath, 'Default', 'Preferences'); - if (Fs.existsSync(preferences)) { - const file = Fs.readFileSync(preferences, 'utf8'); - const data = JSON.parse(file); - data['profile']['exit_type'] = 'none'; - Fs.writeFileSync(preferences, JSON.stringify(data)); + public static async createBrowser(profilePath): Promise { + let args = [ + '--check-for-update-interval=2592000', + '--disable-accelerated-2d-canvas', + '--disable-dev-shm-usage', + '--disable-features=site-per-process', + '--disable-gpu', + '--no-first-run', + '--no-default-browser-check' + ]; + + if (profilePath) { + args.push(`--user-data-dir=${profilePath}`); + + const preferences = urljoin(profilePath, 'Default', 'Preferences'); + if (Fs.existsSync(preferences)) { + const file = Fs.readFileSync(preferences, 'utf8'); + const data = JSON.parse(file); + data['profile']['exit_type'] = 'none'; + Fs.writeFileSync(preferences, JSON.stringify(data)); + } } + puppeteer.use(hidden()); + const browser = await puppeteer.launch({ + args: args, + ignoreHTTPSErrors: true, + headless: false, + defaultViewport: null, + executablePath: executablePath(), + ignoreDefaultArgs: ['--enable-automation', '--enable-blink-features=IdleDetection'] + }); + return browser; } - const browser = await puppeteer.launch({ - args: args, - ignoreHTTPSErrors: true, - headless: false, - defaultViewport: null, - ignoreDefaultArgs: ['--enable-automation', '--enable-blink-features=IdleDetection'] - }); - return browser; -} + /** + * Return the HTML of bot default.gbui. + */ + public static async getHTML(min: GBMinInstance) { + const url = urljoin(GBServer.globals.publicAddress, min.botId); + const browser = await GBSSR.createBrowser(null); + const stylesheetContents = {}; + let html; -async function recursiveFindInFrames (inputFrame, selector) { - const frames = inputFrame.childFrames(); - const results = await Promise.all( - frames.map(async frame => { - const el = await frame.$(selector); - if (el) return el; - if (frame.childFrames().length > 0) { - return await recursiveFindInFrames(frame, selector); - } - return null; - }) - ); - return results.find(Boolean); -} - -/** - * https://developers.google.com/web/tools/puppeteer/articles/ssr#reuseinstance - * @param {string} url URL to prerender. - */ -async function ssr (url: string, useCache: boolean, cacheRefreshRate: number) { - if (RENDER_CACHE.has(url) && useCache) { - const cached = RENDER_CACHE.get(url); - if (Date.now() - cached.renderedAt > cacheRefreshRate && !(cacheRefreshRate <= 0)) { - RENDER_CACHE.delete(url); - } else { - return { - html: cached.html, - status: 200 - }; - } - } - const browser = await createBrowser(null); - const stylesheetContents = {}; - - try { - const page = await browser.newPage(); - await page.setUserAgent( - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' - ); - await page.setRequestInterception(true); - page.on('request', request => { - const requestUrl = request - .url() - .split('?')[0] - .split('#')[0]; - if ( - blockedResourceTypes.indexOf(request.resourceType()) !== -1 || - skippedResources.some(resource => requestUrl.indexOf(resource) !== -1) - ) { - request.abort(); - } else { - request.continue(); - } - }); - - page.on('response', async resp => { - const responseUrl = resp.url(); - const sameOrigin = new URL(responseUrl).origin === new URL(url).origin; - const isStylesheet = resp.request().resourceType() === 'stylesheet'; - if (sameOrigin && isStylesheet) { - stylesheetContents[responseUrl] = await resp.text(); - } - }); - - const response = await page.goto(url, { - timeout: 120000, - waitUntil: 'networkidle0' - }); - - const sleep = ms => { - return new Promise(resolve => { - setTimeout(resolve, ms); + try { + const page = await browser.newPage(); + await page.setUserAgent( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' + ); + await page.setRequestInterception(true); + page.on('request', request => { + const requestUrl = request.url().split('?')[0].split('#')[0]; + if ( + GBSSR.blockedResourceTypes.indexOf(request.resourceType()) !== -1 || + GBSSR.skippedResources.some(resource => requestUrl.indexOf(resource) !== -1) + ) { + request.abort(); + } else { + request.continue(); + } }); - }; - await sleep(45000); - // Inject on page to relative resources load properly. - await page.evaluate(url => { - const base = document.createElement('base'); - base.href = url; - // Add to top of head, before all other resources. - document.head.prepend(base); - }, url); - - // Remove scripts and html imports. They've already executed. - await page.evaluate(() => { - const elements = document.querySelectorAll('script, link[rel="import"]'); - elements.forEach(e => { - e.remove(); + page.on('response', async resp => { + const responseUrl = resp.url(); + const sameOrigin = new URL(responseUrl).origin === new URL(url).origin; + const isStylesheet = resp.request().resourceType() === 'stylesheet'; + if (sameOrigin && isStylesheet) { + stylesheetContents[responseUrl] = await resp.text(); + } }); - }); - // Replace stylesheets in the page with their equivalent