271 lines
12 KiB
JavaScript
271 lines
12 KiB
JavaScript
/*
|
|
* (c) Copyright Ascensio System SIA 2010-2024
|
|
*
|
|
* This program is a free software product. You can redistribute it and/or
|
|
* modify it under the terms of the GNU Affero General Public License (AGPL)
|
|
* version 3 as published by the Free Software Foundation. In accordance with
|
|
* Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
|
|
* that Ascensio System SIA expressly excludes the warranty of non-infringement
|
|
* of any third-party rights.
|
|
*
|
|
* This program is distributed WITHOUT ANY WARRANTY; without even the implied
|
|
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For
|
|
* details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
|
|
*
|
|
* You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish
|
|
* street, Riga, Latvia, EU, LV-1050.
|
|
*
|
|
* The interactive user interfaces in modified source and object code versions
|
|
* of the Program must display Appropriate Legal Notices, as required under
|
|
* Section 5 of the GNU AGPL version 3.
|
|
*
|
|
* Pursuant to Section 7(b) of the License you must retain the original Product
|
|
* logo when distributing the program. Pursuant to Section 7(e) we decline to
|
|
* grant you any rights under trademark law for use of our trademarks.
|
|
*
|
|
* All the Product's GUI elements, including illustrations and icon sets, as
|
|
* well as technical writing content are licensed under the terms of the
|
|
* Creative Commons Attribution-ShareAlike 4.0 International. See the License
|
|
* terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
|
|
*
|
|
*/
|
|
|
|
'use strict';
|
|
const os = require('os');
|
|
const cluster = require('cluster');
|
|
const path = require('path');
|
|
const crypto = require('crypto');
|
|
const config = require('config');
|
|
const utils = require('../utils');
|
|
const commonDefines = require('../commondefines');
|
|
const constants = require('../constants');
|
|
const ms = require('ms');
|
|
const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.sessionabsolute'));
|
|
const cfgCacheStorage = config.get('storage');
|
|
const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage'));
|
|
|
|
// Stubs are needed until integrators pass these parameters to all requests
|
|
let shardKeyCached;
|
|
let wopiSrcCached;
|
|
|
|
const cacheStorage = require('./' + cfgCacheStorage.name);
|
|
const persistentStorage = require('./' + cfgPersistentStorage.name);
|
|
const tenantManager = require('../tenantManager');
|
|
|
|
const HEALTH_CHECK_KEY_MAX = 10000;
|
|
|
|
function getStoragePath(ctx, strPath, opt_specialDir) {
|
|
opt_specialDir = opt_specialDir || cfgCacheStorage.cacheFolderName;
|
|
return opt_specialDir + '/' + tenantManager.getTenantPathPrefix(ctx) + strPath.replace(/\\/g, '/');
|
|
}
|
|
function getStorage(opt_specialDir) {
|
|
return opt_specialDir && opt_specialDir !== cfgCacheStorage.cacheFolderName ? persistentStorage : cacheStorage;
|
|
}
|
|
function getStorageCfg(ctx, opt_specialDir) {
|
|
return opt_specialDir && opt_specialDir !== cfgCacheStorage.cacheFolderName ? cfgPersistentStorage : cfgCacheStorage;
|
|
}
|
|
function canCopyBetweenStorage(storageCfgSrc, storageCfgDst) {
|
|
return storageCfgSrc.name === storageCfgDst.name && storageCfgSrc.endpoint === storageCfgDst.endpoint;
|
|
}
|
|
function isDifferentPersistentStorage() {
|
|
return !canCopyBetweenStorage(cfgCacheStorage, cfgPersistentStorage);
|
|
}
|
|
|
|
async function headObject(ctx, strPath, opt_specialDir) {
|
|
const storage = getStorage(opt_specialDir);
|
|
const storageCfg = getStorageCfg(ctx, opt_specialDir);
|
|
return await storage.headObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir));
|
|
}
|
|
async function getObject(ctx, strPath, opt_specialDir) {
|
|
const storage = getStorage(opt_specialDir);
|
|
const storageCfg = getStorageCfg(ctx, opt_specialDir);
|
|
return await storage.getObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir));
|
|
}
|
|
async function createReadStream(ctx, strPath, opt_specialDir) {
|
|
const storage = getStorage(opt_specialDir);
|
|
const storageCfg = getStorageCfg(ctx, opt_specialDir);
|
|
return await storage.createReadStream(storageCfg, getStoragePath(ctx, strPath, opt_specialDir));
|
|
}
|
|
async function putObject(ctx, strPath, buffer, contentLength, opt_specialDir) {
|
|
const storage = getStorage(opt_specialDir);
|
|
const storageCfg = getStorageCfg(ctx, opt_specialDir);
|
|
return await storage.putObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), buffer, contentLength);
|
|
}
|
|
async function uploadObject(ctx, strPath, filePath, opt_specialDir) {
|
|
const storage = getStorage(opt_specialDir);
|
|
const storageCfg = getStorageCfg(ctx, opt_specialDir);
|
|
return await storage.uploadObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), filePath);
|
|
}
|
|
async function copyObject(ctx, sourceKey, destinationKey, opt_specialDirSrc, opt_specialDirDst) {
|
|
const storageSrc = getStorage(opt_specialDirSrc);
|
|
const storagePathSrc = getStoragePath(ctx, sourceKey, opt_specialDirSrc);
|
|
const storagePathDst = getStoragePath(ctx, destinationKey, opt_specialDirDst);
|
|
const storageCfgSrc = getStorageCfg(ctx, opt_specialDirSrc);
|
|
const storageCfgDst = getStorageCfg(ctx, opt_specialDirDst);
|
|
if (canCopyBetweenStorage(storageCfgSrc, storageCfgDst)) {
|
|
return await storageSrc.copyObject(storageCfgSrc, storageCfgDst, storagePathSrc, storagePathDst);
|
|
} else {
|
|
const storageDst = getStorage(opt_specialDirDst);
|
|
//todo stream
|
|
const buffer = await storageSrc.getObject(storageCfgSrc, storagePathSrc);
|
|
return await storageDst.putObject(storageCfgDst, storagePathDst, buffer, buffer.length);
|
|
}
|
|
}
|
|
async function copyPath(ctx, sourcePath, destinationPath, opt_specialDirSrc, opt_specialDirDst) {
|
|
const list = await listObjects(ctx, sourcePath, opt_specialDirSrc);
|
|
await Promise.all(
|
|
list.map(curValue => {
|
|
return copyObject(ctx, curValue, destinationPath + '/' + getRelativePath(sourcePath, curValue), opt_specialDirSrc, opt_specialDirDst);
|
|
})
|
|
);
|
|
}
|
|
async function listObjects(ctx, strPath, opt_specialDir) {
|
|
const storage = getStorage(opt_specialDir);
|
|
const storageCfg = getStorageCfg(ctx, opt_specialDir);
|
|
const prefix = getStoragePath(ctx, '', opt_specialDir);
|
|
try {
|
|
const list = await storage.listObjects(storageCfg, getStoragePath(ctx, strPath, opt_specialDir));
|
|
return list.map(currentValue => {
|
|
return currentValue.substring(prefix.length);
|
|
});
|
|
} catch (e) {
|
|
ctx.logger.error('storage.listObjects: %s', e.stack);
|
|
return [];
|
|
}
|
|
}
|
|
async function deleteObject(ctx, strPath, opt_specialDir) {
|
|
const storage = getStorage(opt_specialDir);
|
|
const storageCfg = getStorageCfg(ctx, opt_specialDir);
|
|
return await storage.deleteObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir));
|
|
}
|
|
async function deletePath(ctx, strPath, opt_specialDir) {
|
|
const storage = getStorage(opt_specialDir);
|
|
const storageCfg = getStorageCfg(ctx, opt_specialDir);
|
|
return await storage.deletePath(storageCfg, getStoragePath(ctx, strPath, opt_specialDir));
|
|
}
|
|
async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir, useDirectStorageUrls) {
|
|
const storage = getStorage(opt_specialDir);
|
|
const storageCfg = getStorageCfg(ctx, opt_specialDir);
|
|
const storagePath = getStoragePath(ctx, strPath, opt_specialDir);
|
|
const directUrlsEnabled = useDirectStorageUrls ?? storageCfg.useDirectStorageUrls;
|
|
|
|
if (directUrlsEnabled && storage.getDirectSignedUrl) {
|
|
return await storage.getDirectSignedUrl(ctx, storageCfg, baseUrl, storagePath, urlType, optFilename, opt_creationDate);
|
|
} else {
|
|
const storageSecretString = storageCfg.fs.secretString;
|
|
const storageUrlExpires = storageCfg.fs.urlExpires;
|
|
//use fixed bucket name because it hard-coded in nginx
|
|
const bucketName = storageCfg.name === 'storage-fs' ? 'cache' : 'storage-cache';
|
|
const storageFolderName = storageCfg.storageFolderName;
|
|
//replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path
|
|
const userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, '%2f')) : path.basename(strPath);
|
|
const uri = '/' + bucketName + '/' + storageFolderName + '/' + storagePath + '/' + userFriendlyName;
|
|
//RFC 1123 does not allow underscores https://stackoverflow.com/questions/2180465/can-domain-name-subdomains-have-an-underscore-in-it
|
|
let url = utils.checkBaseUrl(ctx, baseUrl, storageCfg).replace(/_/g, '%5f');
|
|
url += uri;
|
|
|
|
const date = Date.now();
|
|
const creationDate = opt_creationDate || date;
|
|
const expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : storageUrlExpires) || 31536000;
|
|
//todo creationDate can be greater because mysql CURRENT_TIMESTAMP uses local time, not UTC
|
|
let expires = creationDate + Math.ceil(Math.abs(date - creationDate) / expiredAfter) * expiredAfter;
|
|
expires = Math.ceil(expires / 1000);
|
|
expires += expiredAfter;
|
|
let md5 = crypto
|
|
.createHash('md5')
|
|
.update(expires + decodeURIComponent(uri) + storageSecretString)
|
|
.digest('base64');
|
|
md5 = md5.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
|
|
|
|
url += '?md5=' + encodeURIComponent(md5);
|
|
url += '&expires=' + encodeURIComponent(expires);
|
|
if (ctx.shardKey) {
|
|
shardKeyCached = ctx.shardKey;
|
|
url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(ctx.shardKey)}`;
|
|
} else if (ctx.wopiSrc) {
|
|
wopiSrcCached = ctx.wopiSrc;
|
|
url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(ctx.wopiSrc)}`;
|
|
} else if (shardKeyCached) {
|
|
//Add stubs for shardkey params until integrators pass these parameters to all requests
|
|
url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(shardKeyCached)}`;
|
|
} else if (wopiSrcCached) {
|
|
url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(wopiSrcCached)}`;
|
|
} else if (process.env.DEFAULT_SHARD_KEY) {
|
|
//todo in fact DEFAULT_SHARD_KEY it's not present in shard map
|
|
//Set DEFAULT_SHARD_KEY from environment as shardkey in case of integrator did not pass this param
|
|
url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(process.env.DEFAULT_SHARD_KEY)}`;
|
|
}
|
|
url += '&filename=' + userFriendlyName;
|
|
return url;
|
|
}
|
|
}
|
|
async function getSignedUrls(ctx, baseUrl, strPath, urlType, opt_creationDate, opt_specialDir) {
|
|
const list = await listObjects(ctx, strPath, opt_specialDir);
|
|
const outputMap = {};
|
|
for (let i = 0; i < list.length; ++i) {
|
|
outputMap[getRelativePath(strPath, list[i])] = await getSignedUrl(ctx, baseUrl, list[i], urlType, undefined, opt_creationDate, opt_specialDir);
|
|
}
|
|
return outputMap;
|
|
}
|
|
async function getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir) {
|
|
return await Promise.all(
|
|
list.map(curValue => {
|
|
return getSignedUrl(ctx, baseUrl, curValue, urlType, undefined, undefined, opt_specialDir);
|
|
})
|
|
);
|
|
}
|
|
async function getSignedUrlsByArray(ctx, baseUrl, list, optPath, urlType, opt_specialDir) {
|
|
const urls = await getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir);
|
|
const outputMap = {};
|
|
for (let i = 0; i < list.length && i < urls.length; ++i) {
|
|
if (optPath) {
|
|
const storagePathSrc = getStoragePath(ctx, optPath, opt_specialDir);
|
|
outputMap[getRelativePath(storagePathSrc, list[i])] = urls[i];
|
|
} else {
|
|
outputMap[list[i]] = urls[i];
|
|
}
|
|
}
|
|
return outputMap;
|
|
}
|
|
function getRelativePath(strBase, strPath) {
|
|
return strPath.substring(strBase.length + 1);
|
|
}
|
|
async function healthCheck(ctx, opt_specialDir) {
|
|
const clusterId = cluster.isWorker ? cluster.worker.id : '';
|
|
const tempName = 'hc_' + os.hostname() + '_' + clusterId + '_' + Math.round(Math.random() * HEALTH_CHECK_KEY_MAX);
|
|
const tempBuffer = Buffer.from([1, 2, 3, 4, 5]);
|
|
try {
|
|
//It's proper to putObject one tempName
|
|
await putObject(ctx, tempName, tempBuffer, tempBuffer.length, opt_specialDir);
|
|
//try to prevent case, when another process can remove same tempName
|
|
await deleteObject(ctx, tempName, opt_specialDir);
|
|
} catch (err) {
|
|
ctx.logger.warn('healthCheck storage(%s) error %s', opt_specialDir, err.stack);
|
|
}
|
|
}
|
|
function needServeStatic(opt_specialDir) {
|
|
const storage = getStorage(opt_specialDir);
|
|
return storage.needServeStatic();
|
|
}
|
|
|
|
module.exports = {
|
|
headObject,
|
|
getObject,
|
|
createReadStream,
|
|
putObject,
|
|
uploadObject,
|
|
copyObject,
|
|
copyPath,
|
|
listObjects,
|
|
deleteObject,
|
|
deletePath,
|
|
getSignedUrl,
|
|
getSignedUrls,
|
|
getSignedUrlsArrayByArray,
|
|
getSignedUrlsByArray,
|
|
getRelativePath,
|
|
isDifferentPersistentStorage,
|
|
healthCheck,
|
|
needServeStatic
|
|
};
|