/* * (c) Copyright Ascensio System SIA 2010-2024 * * This program is a free software product. You can redistribute it and/or * modify it under the terms of the GNU Affero General Public License (AGPL) * version 3 as published by the Free Software Foundation. In accordance with * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect * that Ascensio System SIA expressly excludes the warranty of non-infringement * of any third-party rights. * * This program is distributed WITHOUT ANY WARRANTY; without even the implied * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html * * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish * street, Riga, Latvia, EU, LV-1050. * * The interactive user interfaces in modified source and object code versions * of the Program must display Appropriate Legal Notices, as required under * Section 5 of the GNU AGPL version 3. * * Pursuant to Section 7(b) of the License you must retain the original Product * logo when distributing the program. Pursuant to Section 7(e) we decline to * grant you any rights under trademark law for use of our trademarks. * * All the Product's GUI elements, including illustrations and icon sets, as * well as technical writing content are licensed under the terms of the * Creative Commons Attribution-ShareAlike 4.0 International. See the License * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode * */ 'use strict'; const os = require('os'); const path = require('path'); const fs = require('fs'); const url = require('url'); const co = require('co'); const config = require('config'); const spawnAsync = require('@expo/spawn-async'); const bytes = require('bytes'); const lcid = require('lcid'); const ms = require('ms'); const {pipeline} = require('node:stream/promises'); const commonDefines = require('./../../Common/sources/commondefines'); const storage = require('./../../Common/sources/storage/storage-base'); const utils = require('./../../Common/sources/utils'); const constants = require('./../../Common/sources/constants'); const baseConnector = require('../../DocService/sources/databaseConnectors/baseConnector'); const wopiUtils = require('./../../DocService/sources/wopiUtils'); const taskResult = require('./../../DocService/sources/taskresult'); const statsDClient = require('./../../Common/sources/statsdclient'); const queueService = require('./../../Common/sources/taskqueueRabbitMQ'); const formatChecker = require('./../../Common/sources/formatchecker'); const operationContext = require('./../../Common/sources/operationContext'); const tenantManager = require('./../../Common/sources/tenantManager'); const cfgMaxDownloadBytes = config.get('FileConverter.converter.maxDownloadBytes'); const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout'); const cfgDownloadAttemptMaxCount = config.get('FileConverter.converter.downloadAttemptMaxCount'); const cfgDownloadAttemptDelay = config.get('FileConverter.converter.downloadAttemptDelay'); const cfgFontDir = config.get('FileConverter.converter.fontDir'); const cfgPresentationThemesDir = config.get('FileConverter.converter.presentationThemesDir'); const cfgX2tPath = config.get('FileConverter.converter.x2tPath'); const cfgDocbuilderPath = config.get('FileConverter.converter.docbuilderPath'); const cfgArgs = config.get('FileConverter.converter.args'); const cfgSpawnOptions = config.util.cloneDeep(config.get('FileConverter.converter.spawnOptions')); const cfgErrorFiles = config.get('FileConverter.converter.errorfiles'); const cfgInputLimits = config.get('FileConverter.converter.inputLimits'); const cfgStreamWriterBufferSize = config.get('FileConverter.converter.streamWriterBufferSize'); //cfgMaxRequestChanges was obtained as a result of the test: 84408 changes - 5,16 MB const cfgMaxRequestChanges = config.get('services.CoAuthoring.server.maxRequestChanges'); const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles'); const cfgForgottenFilesName = config.get('services.CoAuthoring.server.forgottenfilesname'); const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate'); const cfgEditor = config.get('services.CoAuthoring.editor'); const cfgRequesFilteringAgent = config.util.cloneDeep(config.get('services.CoAuthoring.request-filtering-agent')); const cfgExternalRequestDirectIfIn = config.get('externalRequest.directIfIn'); const cfgExternalRequestAction = config.get('externalRequest.action'); //windows limit 512(2048) https://msdn.microsoft.com/en-us/library/6e3b887c.aspx //Ubuntu 14.04 limit 4096 http://underyx.me/2015/05/18/raising-the-maximum-number-of-file-descriptors.html //MacOs limit 2048 http://apple.stackexchange.com/questions/33715/too-many-open-files const MAX_OPEN_FILES = 200; const TEMP_PREFIX = 'ASC_CONVERT'; let queue = null; const clientStatsD = statsDClient.getClient(); const exitCodesReturn = [ constants.CONVERT_PARAMS, constants.CONVERT_NEED_PARAMS, constants.CONVERT_CORRUPTED, constants.CONVERT_DRM, constants.CONVERT_DRM_UNSUPPORTED, constants.CONVERT_PASSWORD, constants.CONVERT_LIMITS, constants.CONVERT_DETECT ]; const exitCodesMinorError = [constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM, constants.CONVERT_DRM_UNSUPPORTED, constants.CONVERT_PASSWORD]; const exitCodesUpload = [ constants.NO_ERROR, constants.CONVERT_CORRUPTED, constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM, constants.CONVERT_DRM_UNSUPPORTED ]; const exitCodesCopyOrigin = [constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM]; let inputLimitsXmlCache; function TaskQueueDataConvert(ctx, task) { const cmd = task.getCmd(); this.key = cmd.getDocId(); if (cmd.getSaveKey()) { this.key += cmd.getSaveKey(); } this.fileFrom = null; this.fileTo = null; this.title = cmd.getTitle(); if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA !== cmd.getOutputFormat()) { this.formatTo = cmd.getOutputFormat(); } else { this.formatTo = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF; this.isPDFA = true; } this.csvTxtEncoding = cmd.getCodepage(); this.csvDelimiter = cmd.getDelimiter(); this.csvDelimiterChar = cmd.getDelimiterChar(); this.paid = task.getPaid(); this.embeddedFonts = cmd.embeddedfonts; this.fromChanges = task.getFromChanges(); //todo const tenFontDir = ctx.getCfg('FileConverter.converter.fontDir', cfgFontDir); if (tenFontDir) { this.fontDir = path.resolve(tenFontDir); } else { this.fontDir = null; } const tenPresentationThemesDir = ctx.getCfg('FileConverter.converter.presentationThemesDir', cfgPresentationThemesDir); this.themeDir = path.resolve(tenPresentationThemesDir); this.mailMergeSend = cmd.mailmergesend; this.thumbnail = cmd.thumbnail; this.textParams = cmd.getTextParams(); this.jsonParams = JSON.stringify(cmd.getJsonParams()); this.lcid = cmd.getLCID(); this.password = cmd.getPassword(); this.savePassword = cmd.getSavePassword(); this.noBase64 = cmd.getNoBase64(); this.convertToOrigin = cmd.getConvertToOrigin(); this.oformAsPdf = cmd.getOformAsPdf(); this.timestamp = new Date(); } TaskQueueDataConvert.prototype = { serialize(ctx, fsPath) { let xml = '\ufeff'; xml += ' 0) { xml += this.serializeXmlProp('allowList', allowList.join(';')); } xml += this.serializeXmlProp('allowNetworkRequest', allowNetworkRequest); xml += this.serializeXmlProp('allowPrivateIP', allowPrivateIP); if (proxyUrl) { xml += this.serializeXmlProp('proxy', proxyUrl); } if (proxyUser) { const user = proxyUser.username; const pass = proxyUser.password; xml += this.serializeXmlProp('proxyUser', `${user}:${pass}`); } const proxyHeadersStr = []; for (const name in proxyHeaders) { proxyHeadersStr.push(`${name}:${proxyHeaders[name]}`); } if (proxyHeadersStr.length > 0) { xml += this.serializeXmlProp('proxyHeader', proxyHeadersStr.join(';')); } if (undefined !== oformAsPdf) { xml += this.serializeXmlProp('oformAsPdf', oformAsPdf); } xml += ''; return xml; }, serializeMailMerge(data) { let xml = ''; xml += this.serializeXmlProp('from', data.getFrom()); xml += this.serializeXmlProp('to', data.getTo()); xml += this.serializeXmlProp('subject', data.getSubject()); xml += this.serializeXmlProp('mailFormat', data.getMailFormat()); xml += this.serializeXmlProp('fileName', data.getFileName()); xml += this.serializeXmlProp('message', data.getMessage()); xml += this.serializeXmlProp('recordFrom', data.getRecordFrom()); xml += this.serializeXmlProp('recordTo', data.getRecordTo()); xml += this.serializeXmlProp('recordCount', data.getRecordCount()); xml += this.serializeXmlProp('userid', data.getUserId()); xml += this.serializeXmlProp('url', data.getUrl()); xml += ''; return xml; }, serializeThumbnail(data) { let xml = ''; xml += this.serializeXmlProp('format', data.getFormat()); xml += this.serializeXmlProp('aspect', data.getAspect()); xml += this.serializeXmlProp('first', data.getFirst()); xml += this.serializeXmlProp('width', data.getWidth()); xml += this.serializeXmlProp('height', data.getHeight()); xml += ''; return xml; }, serializeTextParams(data) { let xml = ''; xml += this.serializeXmlProp('m_nTextAssociationType', data.getAssociation()); xml += ''; return xml; }, serializeLimit(ctx) { if (!inputLimitsXmlCache) { let xml = ''; const tenInputLimits = ctx.getCfg('FileConverter.converter.inputLimits', cfgInputLimits); for (let i = 0; i < tenInputLimits.length; ++i) { const limit = tenInputLimits[i]; if (limit.type && limit.zip) { xml += ''; xml += utils.encodeXml(value.toString()); xml += ''; } else { xml += '<' + name + ' xsi:nil="true" />'; } return xml; }, serializeXmlAttr(name, value) { let xml = ''; if (null != value) { xml += ' ' + name + '="'; xml += utils.encodeXml(value.toString()); xml += '"'; } return xml; } }; function getTempDir() { const tempDir = os.tmpdir(); const now = new Date(); let newTemp; while (!newTemp || fs.existsSync(newTemp)) { const newName = [TEMP_PREFIX, now.getFullYear(), now.getMonth(), now.getDate(), '-', (Math.random() * 0x100000000 + 1).toString(36)].join(''); newTemp = path.join(tempDir, newName); } fs.mkdirSync(newTemp); const sourceDir = path.join(newTemp, 'source'); fs.mkdirSync(sourceDir); const resultDir = path.join(newTemp, 'result'); fs.mkdirSync(resultDir); return {temp: newTemp, source: sourceDir, result: resultDir}; } function* isUselessConvertion(ctx, task, cmd) { if (task.getFromChanges() && 'sfc' === cmd.getCommand()) { const selectRes = yield taskResult.select(ctx, cmd.getDocId()); const row = selectRes.length > 0 ? selectRes[0] : null; if (utils.isUselesSfc(row, cmd)) { ctx.logger.warn('isUselessConvertion return true. row=%j', row); return constants.CONVERT_PARAMS; } } return constants.NO_ERROR; } async function changeFormatToExtendedPdf(ctx, dataConvert, cmd) { const originFormat = cmd.getOriginFormat(); const isOriginFormatWithForms = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === originFormat || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM === originFormat || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCXF === originFormat; const isFormatToPdf = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === dataConvert.formatTo || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === dataConvert.formatTo; if (isFormatToPdf && isOriginFormatWithForms) { const format = await formatChecker.getDocumentFormatByFile(dataConvert.fileFrom); if (constants.AVS_OFFICESTUDIO_FILE_CANVAS_WORD === format) { ctx.logger.debug('change format to extended pdf'); dataConvert.formatTo = constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM_PDF; } } } /** * Replace an empty file with a localized template of the requested format, if available. * @param {operationContext} ctx - Operation context for config and logging * @param {string} fileFrom - Path to the file to check/replace * @param {string} ext - Target extension (e.g., 'docx', 'xlsx', 'pptx') * @param {number} _lcid - Optional locale identifier to select template locale */ function replaceEmptyFile(ctx, fileFrom, ext, _lcid) { const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); if (!fs.existsSync(fileFrom) || 0 === fs.lstatSync(fileFrom).size) { let locale = constants.TEMPLATES_DEFAULT_LOCALE; if (_lcid) { let localeNew = lcid.from(_lcid); if (localeNew) { localeNew = localeNew.replace(/_/g, '-'); if (fs.existsSync(path.join(tenNewFileTemplate, localeNew))) { locale = localeNew; } else { ctx.logger.debug('replaceEmptyFile empty locale dir locale=%s', localeNew); } } } const fileTemplatePath = path.join(tenNewFileTemplate, locale, 'new.'); if (fs.existsSync(fileTemplatePath + ext)) { ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); fs.copyFileSync(fileTemplatePath + ext, fileFrom); } else { const format = formatChecker.getFormatFromString(ext); let editorFormat; if (formatChecker.isDocumentFormat(format)) { editorFormat = 'docx'; } else if (formatChecker.isSpreadsheetFormat(format)) { editorFormat = 'xlsx'; } else if (formatChecker.isPresentationFormat(format)) { editorFormat = 'pptx'; } if (fs.existsSync(fileTemplatePath + editorFormat)) { ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); fs.copyFileSync(fileTemplatePath + editorFormat, fileFrom); } } } } function* downloadFile(ctx, uri, fileFrom, withAuthorization, isInJwtToken, opt_headers) { const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout); const tenDownloadAttemptMaxCount = ctx.getCfg('FileConverter.converter.downloadAttemptMaxCount', cfgDownloadAttemptMaxCount); const tenDownloadAttemptDelay = ctx.getCfg('FileConverter.converter.downloadAttemptDelay', cfgDownloadAttemptDelay); let res = constants.CONVERT_DOWNLOAD; let data = null; let sha256 = null; let downloadAttemptCount = 0; const urlParsed = url.parse(uri); const filterStatus = yield* utils.checkHostFilter(ctx, urlParsed.hostname); if (0 == filterStatus) { while (constants.NO_ERROR !== res && downloadAttemptCount++ < tenDownloadAttemptMaxCount) { try { let authorization; if (utils.canIncludeOutboxAuthorization(ctx, uri) && withAuthorization) { const secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox); authorization = utils.fillJwtForRequest(ctx, {url: uri}, secret, false); } const getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, tenMaxDownloadBytes, authorization, isInJwtToken, opt_headers); data = getRes.body; sha256 = getRes.sha256; res = constants.NO_ERROR; } catch (err) { res = constants.CONVERT_DOWNLOAD; ctx.logger.error('error downloadFile:url=%s;attempt=%d;code:%s;connect:%s %s', uri, downloadAttemptCount, err.code, err.connect, err.stack); //not continue attempts if timeout if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { break; } else if (err.code === 'EMSGSIZE') { res = constants.CONVERT_LIMITS; break; } else { yield utils.sleep(tenDownloadAttemptDelay); } } } if (constants.NO_ERROR === res) { ctx.logger.debug('downloadFile complete filesize=%d sha256=%s', data.length, sha256); fs.writeFileSync(fileFrom, data); } } else { ctx.logger.error('checkIpFilter error:url=%s;code:%s;', uri, filterStatus); res = constants.CONVERT_DOWNLOAD; } return res; } function* downloadFileFromStorage(ctx, strPath, dir, opt_specialDir) { const list = yield storage.listObjects(ctx, strPath, opt_specialDir); ctx.logger.debug('downloadFileFromStorage list %s', list.toString()); //create dirs const dirsToCreate = []; const dirStruct = {}; list.forEach(file => { let curDirPath = dir; const curDirStruct = dirStruct; const parts = storage.getRelativePath(strPath, file).split('/'); for (let i = 0; i < parts.length - 1; ++i) { const part = parts[i]; curDirPath = path.join(curDirPath, part); if (!curDirStruct[part]) { curDirStruct[part] = {}; dirsToCreate.push(curDirPath); } } }); //make dirs for (let i = 0; i < dirsToCreate.length; ++i) { fs.mkdirSync(dirsToCreate[i]); } //download //todo Promise.all for (let i = 0; i < list.length; ++i) { const file = list[i]; const fileRel = storage.getRelativePath(strPath, file); const data = yield storage.getObject(ctx, file, opt_specialDir); fs.writeFileSync(path.join(dir, fileRel), data); } return list.length; } function* processDownloadFromStorage(ctx, dataConvert, cmd, task, tempDirs, authorProps) { const tenEditor = ctx.getCfg('services.CoAuthoring.editor', cfgEditor); let res = constants.NO_ERROR; let concatDir; let concatTemplate; if (task.getFromOrigin() || task.getFromSettings()) { if (task.getFromChanges()) { const changesDir = path.join(tempDirs.source, constants.CHANGES_NAME); fs.mkdirSync(changesDir); let filesCount = 0; if (cmd.getSaveKey()) { filesCount = yield* downloadFileFromStorage(ctx, cmd.getDocId() + cmd.getSaveKey(), changesDir); } if (filesCount > 0) { concatDir = changesDir; concatTemplate = 'changes0'; } else { dataConvert.fromChanges = false; task.setFromChanges(dataConvert.fromChanges); } } dataConvert.fileFrom = path.join(tempDirs.source, 'origin.' + cmd.getFormat()); } else { //overwrite some files from m_sKey (for example Editor.bin or changes) if (cmd.getSaveKey()) { yield* downloadFileFromStorage(ctx, cmd.getDocId() + cmd.getSaveKey(), tempDirs.source); } const format = cmd.getFormat() || 'bin'; dataConvert.fileFrom = path.join(tempDirs.source, 'Editor.' + format); concatDir = tempDirs.source; } if (!utils.checkPathTraversal(ctx, dataConvert.key, tempDirs.source, dataConvert.fileFrom)) { return constants.CONVERT_PARAMS; } //mail merge const mailMergeSend = cmd.getMailMergeSend(); if (mailMergeSend) { yield* downloadFileFromStorage(ctx, cmd.getDocId() + mailMergeSend.getJsonKey(), tempDirs.source); concatDir = tempDirs.source; } if (concatDir) { yield* concatFiles(concatDir, concatTemplate); if (concatTemplate) { const filenames = fs.readdirSync(concatDir); filenames.forEach(file => { if (file.match(new RegExp(`${concatTemplate}\\d+\\.`))) { fs.rmSync(path.join(concatDir, file)); } }); } } //todo rework if (!fs.existsSync(dataConvert.fileFrom)) { if (fs.existsSync(path.join(tempDirs.source, 'origin.docx'))) { dataConvert.fileFrom = path.join(tempDirs.source, 'origin.docx'); } else if (fs.existsSync(path.join(tempDirs.source, 'origin.xlsx'))) { dataConvert.fileFrom = path.join(tempDirs.source, 'origin.xlsx'); } else if (fs.existsSync(path.join(tempDirs.source, 'origin.pptx'))) { dataConvert.fileFrom = path.join(tempDirs.source, 'origin.pptx'); } else if (fs.existsSync(path.join(tempDirs.source, 'origin.pdf'))) { dataConvert.fileFrom = path.join(tempDirs.source, 'origin.pdf'); } if (fs.existsSync(dataConvert.fileFrom)) { const fileFromNew = path.join(path.dirname(dataConvert.fileFrom), 'Editor.bin'); fs.renameSync(dataConvert.fileFrom, fileFromNew); dataConvert.fileFrom = fileFromNew; } } yield changeFormatToExtendedPdf(ctx, dataConvert, cmd); if (task.getFromChanges() && !(task.getFromOrigin() || task.getFromSettings())) { const sha256 = yield utils.checksumFile('sha256', dataConvert.fileFrom); if (tenEditor['binaryChanges']) { res = yield* processChangesBin(ctx, tempDirs, task, cmd, authorProps, sha256); } else { res = yield* processChangesBase64(ctx, tempDirs, task, cmd, authorProps, sha256); } } return res; } function* concatFiles(source, template) { template = template || 'Editor'; //concatenate EditorN.ext parts in Editor.ext const list = yield utils.listObjects(source, true); list.sort(utils.compareStringByLength); const createdTargets = new Set(); for (let i = 0; i < list.length; ++i) { const file = list[i]; if (file.match(new RegExp(`${template}\\d+\\.`))) { const target = file.replace(new RegExp(`(${template})\\d+(\\..*)`), '$1$2'); const isFirst = !createdTargets.has(target); const writeOpts = isFirst ? undefined : {flags: 'a'}; const writeStream = yield utils.promiseCreateWriteStream(target, writeOpts); const readStream = yield utils.promiseCreateReadStream(file); // Use raw pipeline for file-to-file operations to surface real errors yield pipeline(readStream, writeStream); if (isFirst) { createdTargets.add(target); } } } } function* processChangesBin(ctx, tempDirs, task, cmd, authorProps, sha256) { const tenStreamWriterBufferSize = ctx.getCfg('FileConverter.converter.streamWriterBufferSize', cfgStreamWriterBufferSize); const tenMaxRequestChanges = ctx.getCfg('services.CoAuthoring.server.maxRequestChanges', cfgMaxRequestChanges); let res = constants.NO_ERROR; const changesDir = path.join(tempDirs.source, constants.CHANGES_NAME); fs.mkdirSync(changesDir); let indexFile = 0; let changesAuthor = null; let changesAuthorUnique = null; let changesIndex = null; const changesHistory = { serverVersion: commonDefines.buildVersion, changes: [] }; const forceSave = cmd.getForceSave(); let forceSaveTime; let forceSaveIndex = Number.MAX_VALUE; if (forceSave && undefined !== forceSave.getTime() && undefined !== forceSave.getIndex()) { forceSaveTime = forceSave.getTime(); forceSaveIndex = forceSave.getIndex(); } const extChangeInfo = cmd.getExternalChangeInfo(); let extChanges; if (extChangeInfo) { extChanges = [ { id: cmd.getDocId(), change_id: 0, change_data: Buffer.alloc(0), user_id: extChangeInfo.user_id, user_id_original: extChangeInfo.user_id_original, user_name: extChangeInfo.user_name, change_date: new Date(extChangeInfo.change_date) } ]; } let streamObj = yield* streamCreateBin(ctx, changesDir, indexFile++, {highWaterMark: tenStreamWriterBufferSize}); yield* streamWriteBin(streamObj, Buffer.from(utils.getChangesFileHeader(), 'utf-8')); let curIndexStart = 0; let curIndexEnd = Math.min(curIndexStart + tenMaxRequestChanges, forceSaveIndex); while (curIndexStart < curIndexEnd || extChanges) { let changes = []; if (curIndexStart < curIndexEnd) { changes = yield baseConnector.getChangesPromise(ctx, cmd.getDocId(), curIndexStart, curIndexEnd, forceSaveTime); if (changes.length > 0 && changes[0].change_data.subarray(0, 'ENCRYPTED;'.length).includes('ENCRYPTED;')) { ctx.logger.warn('processChanges encrypted changes'); //todo sql request instead? res = constants.EDITOR_CHANGES; } res = yield* isUselessConvertion(ctx, task, cmd); if (constants.NO_ERROR !== res) { break; } } if (0 === changes.length && extChanges) { changes = extChanges; } extChanges = undefined; for (let i = 0; i < changes.length; ++i) { const change = changes[i]; if (null === changesAuthor || changesAuthor !== change.user_id_original) { if (null !== changesAuthor) { yield* streamEndBin(streamObj); streamObj = yield* streamCreateBin(ctx, changesDir, indexFile++); yield* streamWriteBin(streamObj, Buffer.from(utils.getChangesFileHeader(), 'utf-8')); } const strDate = baseConnector.getDateTime(change.change_date); changesHistory.changes.push({documentSha256: sha256, created: strDate, user: {id: change.user_id_original, name: change.user_name}}); } changesAuthor = change.user_id_original; changesAuthorUnique = change.user_id; yield* streamWriteBin(streamObj, change.change_data); streamObj.isNoChangesInFile = false; } if (changes.length > 0) { authorProps.lastModifiedBy = changes[changes.length - 1].user_name; authorProps.modified = changes[changes.length - 1].change_date.toISOString().slice(0, 19) + 'Z'; } if (changes.length === curIndexEnd - curIndexStart) { curIndexStart += tenMaxRequestChanges; curIndexEnd = Math.min(curIndexStart + tenMaxRequestChanges, forceSaveIndex); } else { break; } } yield* streamEndBin(streamObj); if (streamObj.isNoChangesInFile) { fs.unlinkSync(streamObj.filePath); } if (null !== changesAuthorUnique) { changesIndex = utils.getIndexFromUserId(changesAuthorUnique, changesAuthor); } if ( null == changesAuthor && null == changesIndex && forceSave && undefined !== forceSave.getAuthorUserId() && undefined !== forceSave.getAuthorUserIndex() ) { changesAuthor = forceSave.getAuthorUserId(); changesIndex = forceSave.getAuthorUserIndex(); } cmd.setUserId(changesAuthor); cmd.setUserIndex(changesIndex); fs.writeFileSync(path.join(tempDirs.result, 'changesHistory.json'), JSON.stringify(changesHistory), 'utf8'); ctx.logger.debug('processChanges end'); return res; } function* streamCreateBin(ctx, changesDir, indexFile, opt_options) { const fileName = constants.CHANGES_NAME + indexFile + '.bin'; const filePath = path.join(changesDir, fileName); const writeStream = yield utils.promiseCreateWriteStream(filePath, opt_options); writeStream.on('error', err => { //todo integrate error handle in main thread (probable: set flag here and check it in main thread) ctx.logger.error('WriteStreamError %s', err.stack); }); return {writeStream, filePath, isNoChangesInFile: true}; } function* streamWriteBin(streamObj, buf) { if (!streamObj.writeStream.write(buf)) { yield utils.promiseWaitDrain(streamObj.writeStream); } } function* streamEndBin(streamObj) { streamObj.writeStream.end(); yield utils.promiseWaitClose(streamObj.writeStream); } function* processChangesBase64(ctx, tempDirs, task, cmd, authorProps, sha256) { const tenStreamWriterBufferSize = ctx.getCfg('FileConverter.converter.streamWriterBufferSize', cfgStreamWriterBufferSize); const tenMaxRequestChanges = ctx.getCfg('services.CoAuthoring.server.maxRequestChanges', cfgMaxRequestChanges); let res = constants.NO_ERROR; const changesDir = path.join(tempDirs.source, constants.CHANGES_NAME); fs.mkdirSync(changesDir); let indexFile = 0; let changesAuthor = null; let changesAuthorUnique = null; let changesIndex = null; const changesHistory = { serverVersion: commonDefines.buildVersion, changes: [] }; const forceSave = cmd.getForceSave(); let forceSaveTime; let forceSaveIndex = Number.MAX_VALUE; if (forceSave && undefined !== forceSave.getTime() && undefined !== forceSave.getIndex()) { forceSaveTime = forceSave.getTime(); forceSaveIndex = forceSave.getIndex(); } const extChangeInfo = cmd.getExternalChangeInfo(); let extChanges; if (extChangeInfo) { extChanges = [ { id: cmd.getDocId(), change_id: 0, change_data: '', user_id: extChangeInfo.user_id, user_id_original: extChangeInfo.user_id_original, user_name: extChangeInfo.user_name, change_date: new Date(extChangeInfo.change_date) } ]; } let streamObj = yield* streamCreate(ctx, changesDir, indexFile++, {highWaterMark: tenStreamWriterBufferSize}); let curIndexStart = 0; let curIndexEnd = Math.min(curIndexStart + tenMaxRequestChanges, forceSaveIndex); while (curIndexStart < curIndexEnd || extChanges) { let changes = []; if (curIndexStart < curIndexEnd) { changes = yield baseConnector.getChangesPromise(ctx, cmd.getDocId(), curIndexStart, curIndexEnd, forceSaveTime); if (changes.length > 0 && changes[0].change_data.startsWith('ENCRYPTED;')) { ctx.logger.warn('processChanges encrypted changes'); //todo sql request instead? res = constants.EDITOR_CHANGES; } res = yield* isUselessConvertion(ctx, task, cmd); if (constants.NO_ERROR !== res) { break; } } if (0 === changes.length && extChanges) { changes = extChanges; } extChanges = undefined; for (let i = 0; i < changes.length; ++i) { const change = changes[i]; if (null === changesAuthor || changesAuthor !== change.user_id_original) { if (null !== changesAuthor) { yield* streamEnd(streamObj, ']'); streamObj = yield* streamCreate(ctx, changesDir, indexFile++); } const strDate = baseConnector.getDateTime(change.change_date); changesHistory.changes.push({documentSha256: sha256, created: strDate, user: {id: change.user_id_original, name: change.user_name}}); yield* streamWrite(streamObj, '['); } else { yield* streamWrite(streamObj, ','); } changesAuthor = change.user_id_original; changesAuthorUnique = change.user_id; yield* streamWrite(streamObj, change.change_data); streamObj.isNoChangesInFile = false; } if (changes.length > 0) { authorProps.lastModifiedBy = changes[changes.length - 1].user_name; authorProps.modified = changes[changes.length - 1].change_date.toISOString().slice(0, 19) + 'Z'; } if (changes.length === curIndexEnd - curIndexStart) { curIndexStart += tenMaxRequestChanges; curIndexEnd = Math.min(curIndexStart + tenMaxRequestChanges, forceSaveIndex); } else { break; } } yield* streamEnd(streamObj, ']'); if (streamObj.isNoChangesInFile) { fs.unlinkSync(streamObj.filePath); } if (null !== changesAuthorUnique) { changesIndex = utils.getIndexFromUserId(changesAuthorUnique, changesAuthor); } if ( null == changesAuthor && null == changesIndex && forceSave && undefined !== forceSave.getAuthorUserId() && undefined !== forceSave.getAuthorUserIndex() ) { changesAuthor = forceSave.getAuthorUserId(); changesIndex = forceSave.getAuthorUserIndex(); } cmd.setUserId(changesAuthor); cmd.setUserIndex(changesIndex); fs.writeFileSync(path.join(tempDirs.result, 'changesHistory.json'), JSON.stringify(changesHistory), 'utf8'); ctx.logger.debug('processChanges end'); return res; } function* streamCreate(ctx, changesDir, indexFile, opt_options) { const fileName = constants.CHANGES_NAME + indexFile + '.json'; const filePath = path.join(changesDir, fileName); const writeStream = yield utils.promiseCreateWriteStream(filePath, opt_options); writeStream.on('error', err => { //todo integrate error handle in main thread (probable: set flag here and check it in main thread) ctx.logger.error('WriteStreamError %s', err.stack); }); return {writeStream, filePath, isNoChangesInFile: true}; } function* streamWrite(streamObj, text) { if (!streamObj.writeStream.write(text, 'utf8')) { yield utils.promiseWaitDrain(streamObj.writeStream); } } function* streamEnd(streamObj, text) { streamObj.writeStream.end(text, 'utf8'); yield utils.promiseWaitClose(streamObj.writeStream); } function* processUploadToStorage(ctx, dir, storagePath, calcChecksum, opt_specialDirDst, opt_ignorPrefix) { let list = yield utils.listObjects(dir); if (opt_ignorPrefix) { list = list.filter(dir => !dir.startsWith(opt_ignorPrefix)); } if (list.length < MAX_OPEN_FILES) { yield* processUploadToStorageChunk(ctx, list, dir, storagePath, calcChecksum, opt_specialDirDst); } else { for (let i = 0, j = list.length; i < j; i += MAX_OPEN_FILES) { yield* processUploadToStorageChunk(ctx, list.slice(i, i + MAX_OPEN_FILES), dir, storagePath, calcChecksum, opt_specialDirDst); } } } function* processUploadToStorageChunk(ctx, list, dir, storagePath, calcChecksum, opt_specialDirDst) { const promises = list.reduce((r, curValue) => { const localValue = storagePath + '/' + curValue.substring(dir.length + 1); let checksum; if (calcChecksum) { checksum = utils.checksumFile('sha256', curValue).then(result => { ctx.logger.debug('processUploadToStorageChunk path=%s; sha256=%s', localValue, result); }); } const upload = storage.uploadObject(ctx, localValue, curValue, opt_specialDirDst); r.push(checksum, upload); return r; }, []); yield Promise.all(promises); } function* processUploadToStorageErrorFile(ctx, dataConvert, tempDirs, childRes, exitCode, exitSignal, error) { const tenErrorFiles = ctx.getCfg('FileConverter.converter.errorfiles', cfgErrorFiles); if (!tenErrorFiles) { return; } let output = ''; if (undefined !== childRes.stdout) { output += `stdout:${childRes.stdout}\n`; } if (undefined !== childRes.stderr) { output += `stderr:${childRes.stderr}\n`; } output += `ExitCode (code=${exitCode};signal=${exitSignal};error:${error})`; const outputPath = path.join(tempDirs.temp, 'console.txt'); fs.writeFileSync(outputPath, output, {encoding: 'utf8'}); //ignore result dir with temp dir inside(see m_sTempDir param) to reduce the amount of data transferred const ignorePrefix = path.normalize(tempDirs.result); const format = path.extname(dataConvert.fileFrom).substring(1) || 'unknown'; yield* processUploadToStorage(ctx, tempDirs.temp, format + '/' + dataConvert.key, false, tenErrorFiles, ignorePrefix); ctx.logger.debug('processUploadToStorage error complete(id=%s)', dataConvert.key); } function writeProcessOutputToLog(ctx, childRes, isDebug) { if (childRes) { if (undefined !== childRes.stdout) { if (isDebug) { ctx.logger.debug('stdout:%s', childRes.stdout); } else { ctx.logger.error('stdout:%s', childRes.stdout); } } if (undefined !== childRes.stderr) { if (isDebug) { ctx.logger.debug('stderr:%s', childRes.stderr); } else { ctx.logger.error('stderr:%s', childRes.stderr); } } } } function* postProcess(ctx, cmd, dataConvert, tempDirs, childRes, error, isTimeout) { let exitCode = 0; let exitSignal = null; if (childRes) { exitCode = childRes.status; exitSignal = childRes.signal; } //CONVERT_CELLLIMITS is not an error, but an indicator that data was lost during opening (can be displayed as an error) if ((0 !== exitCode && constants.CONVERT_CELLLIMITS !== -exitCode) || null !== exitSignal) { if (-1 !== exitCodesReturn.indexOf(-exitCode)) { error = -exitCode; } else if (isTimeout) { error = constants.CONVERT_TIMEOUT; } else { error = constants.CONVERT; } if (-1 !== exitCodesMinorError.indexOf(error)) { writeProcessOutputToLog(ctx, childRes, true); ctx.logger.debug('ExitCode (code=%d;signal=%s;error:%d)', exitCode, exitSignal, error); } else { writeProcessOutputToLog(ctx, childRes, false); ctx.logger.error('ExitCode (code=%d;signal=%s;error:%d)', exitCode, exitSignal, error); yield* processUploadToStorageErrorFile(ctx, dataConvert, tempDirs, childRes, exitCode, exitSignal, error); } } else { writeProcessOutputToLog(ctx, childRes, true); ctx.logger.debug('ExitCode (code=%d;signal=%s;error:%d)', exitCode, exitSignal, error); } if (-1 !== exitCodesUpload.indexOf(error)) { if (-1 !== exitCodesCopyOrigin.indexOf(error)) { const originPath = path.join(path.dirname(dataConvert.fileTo), 'origin' + path.extname(dataConvert.fileFrom)); if (!fs.existsSync(dataConvert.fileTo)) { fs.copyFileSync(dataConvert.fileFrom, originPath); ctx.logger.debug('copyOrigin complete'); } } //todo clarify calcChecksum conditions const calcChecksum = 0 === (constants.AVS_OFFICESTUDIO_FILE_CANVAS & cmd.getOutputFormat()); yield* processUploadToStorage(ctx, tempDirs.result, dataConvert.key, calcChecksum); ctx.logger.debug('processUploadToStorage complete'); } cmd.setStatusInfo(error); let existFile = false; try { existFile = fs.lstatSync(dataConvert.fileTo).isFile(); } catch (_err) { existFile = false; } if (!existFile) { //todo review. the stub in the case of AVS_OFFICESTUDIO_FILE_OTHER_OOXML x2t changes the file extension. const fileToBasename = path.basename(dataConvert.fileTo, path.extname(dataConvert.fileTo)); const fileToDir = path.dirname(dataConvert.fileTo); const files = fs.readdirSync(fileToDir); for (let i = 0; i < files.length; ++i) { const fileCur = files[i]; if (0 == fileCur.indexOf(fileToBasename)) { dataConvert.fileTo = path.join(fileToDir, fileCur); break; } } } cmd.setOutputPath(path.basename(dataConvert.fileTo)); if (!cmd.getTitle()) { cmd.setTitle(cmd.getOutputPath()); } const queueData = new commonDefines.TaskQueueData(); queueData.setCtx(ctx); queueData.setCmd(cmd); ctx.logger.debug('output (data=%j)', queueData); return queueData; } function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken) { const tenX2tPath = ctx.getCfg('FileConverter.converter.x2tPath', cfgX2tPath); const tenDocbuilderPath = ctx.getCfg('FileConverter.converter.docbuilderPath', cfgDocbuilderPath); const tenArgs = ctx.getCfg('FileConverter.converter.args', cfgArgs); let childRes, isTimeout = false; let childArgs; if (tenArgs.length > 0) { childArgs = tenArgs.trim().replace(/ +/g, ' ').split(' '); } else { childArgs = []; } let processPath; if (!builderParams) { processPath = tenX2tPath; const paramsFile = path.join(tempDirs.temp, 'params.xml'); dataConvert.serialize(ctx, paramsFile); childArgs.push(paramsFile); const hiddenXml = yield dataConvert.serializeHidden(ctx); if (hiddenXml) { childArgs.push(hiddenXml); } } else { fs.mkdirSync(path.join(tempDirs.result, 'output')); processPath = tenDocbuilderPath; childArgs.push('--check-fonts=0'); childArgs.push('--save-use-only-names=' + tempDirs.result + '/output'); if (builderParams.argument) { childArgs.push(`--argument=${JSON.stringify(builderParams.argument)}`); } childArgs.push('--options=' + dataConvert.serializeOptions(ctx, isInJwtToken)); childArgs.push(dataConvert.fileFrom); } let timeoutId; try { const tenSpawnOptions = ctx.getCfg('FileConverter.converter.spawnOptions', cfgSpawnOptions); //copy to avoid modification of global cfgSpawnOptions const spawnOptions = Object.assign({}, tenSpawnOptions); spawnOptions.env = Object.assign({}, process.env, spawnOptions.env); if (authorProps.lastModifiedBy && authorProps.modified) { spawnOptions.env['LAST_MODIFIED_BY'] = authorProps.lastModifiedBy; spawnOptions.env['MODIFIED'] = authorProps.modified; } const spawnAsyncPromise = spawnAsync(processPath, childArgs, spawnOptions); childRes = spawnAsyncPromise.child; const waitMS = Math.max(0, task.getVisibilityTimeout() * 1000 - (new Date().getTime() - getTaskTime.getTime())); timeoutId = setTimeout(() => { isTimeout = true; timeoutId = undefined; //close stdio streams to enable emit 'close' event even if HtmlFileInternal is hung-up childRes.stdin.end(); childRes.stdout.destroy(); childRes.stderr.destroy(); childRes.kill(); }, waitMS); childRes = yield spawnAsyncPromise; } catch (err) { if (null === err.status) { ctx.logger.error('error spawnAsync %s', err.stack); } else { ctx.logger.debug('error spawnAsync %s', err.stack); } childRes = err; } if (undefined !== timeoutId) { clearTimeout(timeoutId); } return {childRes, isTimeout}; } function* ExecuteTask(ctx, task) { const tenForgottenFiles = ctx.getCfg('services.CoAuthoring.server.forgottenfiles', cfgForgottenFiles); const tenForgottenFilesName = ctx.getCfg('services.CoAuthoring.server.forgottenfilesname', cfgForgottenFilesName); let startDate = null; let curDate = null; if (clientStatsD) { startDate = curDate = new Date(); } const getTaskTime = new Date(); const cmd = task.getCmd(); const dataConvert = new TaskQueueDataConvert(ctx, task); ctx.logger.info('Start Task'); let error = constants.NO_ERROR; const tempDirs = getTempDir(); const fileTo = task.getToFile(); dataConvert.fileTo = fileTo ? path.join(tempDirs.result, fileTo) : ''; const builderParams = cmd.getBuilderParams(); const authorProps = {lastModifiedBy: null, modified: null}; let isInJwtToken = cmd.getWithAuthorization(); error = yield* isUselessConvertion(ctx, task, cmd); if (constants.NO_ERROR !== error) { ctx.logger.error('constants.NO_ERROR !== error', error); } else if (cmd.getUrl()) { const format = cmd.getFormat(); dataConvert.fileFrom = path.join(tempDirs.source, dataConvert.key + '.' + format); if (utils.checkPathTraversal(ctx, dataConvert.key, tempDirs.source, dataConvert.fileFrom)) { let url = cmd.getUrl(); let withAuthorization = cmd.getWithAuthorization(); let headers; let fileSize; const wopiParams = cmd.getWopiParams(); if (wopiParams) { withAuthorization = false; isInJwtToken = true; const fileInfo = wopiParams.commonInfo?.fileInfo; fileSize = fileInfo?.Size; ({url, headers} = yield wopiUtils.getWopiFileUrl(ctx, fileInfo, wopiParams.userAuth)); } if (undefined === fileSize || fileSize > 0) { error = yield* downloadFile(ctx, url, dataConvert.fileFrom, withAuthorization, isInJwtToken, headers); } if (constants.NO_ERROR === error) { replaceEmptyFile(ctx, dataConvert.fileFrom, format, cmd.getLCID()); } if (clientStatsD) { clientStatsD.timing('conv.downloadFile', new Date() - curDate); curDate = new Date(); } } else { error = constants.CONVERT_PARAMS; } } else if (cmd.getSaveKey() || task.getFromOrigin() || task.getFromSettings()) { yield* downloadFileFromStorage(ctx, cmd.getDocId(), tempDirs.source); ctx.logger.debug('downloadFileFromStorage complete'); if (clientStatsD) { clientStatsD.timing('conv.downloadFileFromStorage', new Date() - curDate); curDate = new Date(); } error = yield* processDownloadFromStorage(ctx, dataConvert, cmd, task, tempDirs, authorProps); } else if (cmd.getForgotten()) { yield* downloadFileFromStorage(ctx, cmd.getForgotten(), tempDirs.source, tenForgottenFiles); ctx.logger.debug('downloadFileFromStorage complete'); const list = yield utils.listObjects(tempDirs.source, false); if (list.length > 0) { dataConvert.fileFrom = list[0]; //store indicator file to determine if opening was from the forgotten file const forgottenMarkPath = tempDirs.result + '/' + tenForgottenFilesName + '.txt'; fs.writeFileSync(forgottenMarkPath, tenForgottenFilesName, {encoding: 'utf8'}); } else { error = constants.UNKNOWN; } } else if (builderParams) { //in cause script in POST body yield* downloadFileFromStorage(ctx, cmd.getDocId(), tempDirs.source); ctx.logger.debug('downloadFileFromStorage complete'); const list = yield utils.listObjects(tempDirs.source, false); if (list.length > 0) { dataConvert.fileFrom = list[0]; } } else { error = constants.UNKNOWN; } let childRes = null; let isTimeout = false; if (constants.NO_ERROR === error) { ({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken)); const canRollback = childRes && 0 !== childRes.status && !isTimeout && task.getFromChanges() && constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML !== dataConvert.formatTo && !formatChecker.isOOXFormat(dataConvert.formatTo) && !formatChecker.isBrowserEditorFormat(dataConvert.formatTo) && !cmd.getWopiParams(); if (canRollback) { ctx.logger.warn( 'rollback to save changes to ooxml. See assemblyFormatAsOrigin param. formatTo=%s', formatChecker.getStringFromFormat(dataConvert.formatTo) ); const extOld = path.extname(dataConvert.fileTo); const extNew = '.' + formatChecker.getStringFromFormat(constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML); dataConvert.formatTo = constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML; dataConvert.fileTo = dataConvert.fileTo.slice(0, -extOld.length) + extNew; ({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken)); } if (clientStatsD) { clientStatsD.timing('conv.spawnSync', new Date() - curDate); curDate = new Date(); } } const resData = yield* postProcess(ctx, cmd, dataConvert, tempDirs, childRes, error, isTimeout); ctx.logger.debug('postProcess'); if (clientStatsD) { clientStatsD.timing('conv.postProcess', new Date() - curDate); curDate = new Date(); } if (tempDirs) { fs.rmSync(tempDirs.temp, {recursive: true, force: true}); ctx.logger.debug('deleteFolderRecursive'); if (clientStatsD) { clientStatsD.timing('conv.deleteFolderRecursive', new Date() - curDate); curDate = new Date(); } } if (clientStatsD) { clientStatsD.timing('conv.allconvert', new Date() - startDate); } ctx.logger.info('End Task'); return resData; } function ackTask(ctx, res, task, ack) { return co(function* () { try { if (!res) { res = createErrorResponse(ctx, task); } if (res) { yield queue.addResponse(res); ctx.logger.info('ackTask addResponse'); } } catch (err) { ctx.logger.error('ackTask %s', err.stack); } finally { ack(); ctx.logger.info('ackTask ack'); } }); } function receiveTaskSetTimeout(ctx, task, ack, outParams) { //add DownloadTimeout to upload results const delay = task.getVisibilityTimeout() * 1000 + ms(cfgDownloadTimeout.wholeCycle); return setTimeout(() => { return co(function* () { outParams.isAck = true; ctx.logger.error('receiveTask timeout %d', delay); yield ackTask(ctx, null, task, ack); yield queue.closeOrWait(); process.exit(1); }); }, delay); } function receiveTask(data, ack) { return co(function* () { let res = null; let task = null; const outParams = {isAck: false}; let timeoutId = undefined; const ctx = new operationContext.Context(); try { task = new commonDefines.TaskQueueData(JSON.parse(data)); if (task) { ctx.initFromTaskQueueData(task); yield ctx.initTenantCache(); timeoutId = receiveTaskSetTimeout(ctx, task, ack, outParams); res = yield* ExecuteTask(ctx, task); } } catch (err) { ctx.logger.error('receiveTask %s', err.stack); } finally { clearTimeout(timeoutId); if (!outParams.isAck) { yield ackTask(ctx, res, task, ack); } } }); } function createErrorResponse(ctx, task) { if (!task) { return null; } ctx.logger.debug('createErrorResponse'); //simulate error response const cmd = task.getCmd(); cmd.setStatusInfo(constants.CONVERT_TEMPORARY); const res = new commonDefines.TaskQueueData(); res.setCtx(ctx); res.setCmd(cmd); return res; } function simulateErrorResponse(data) { const task = new commonDefines.TaskQueueData(JSON.parse(data)); const ctx = new operationContext.Context(); ctx.initFromTaskQueueData(task); //todo //yield ctx.initTenantCache(); return createErrorResponse(ctx, task); } function run() { queue = new queueService(simulateErrorResponse); queue.on('task', receiveTask); queue.init(true, true, true, false, false, false, err => { if (null != err) { operationContext.global.logger.error('createTaskQueue error: %s', err.stack); } }); } exports.run = run;