mirror of
https://github.com/Baboo7/strapi-plugin-import-export-entries.git
synced 2025-09-04 00:02:40 -04:00
chore: Setup typescript for server
This commit is contained in:
parent
fb89c4efcb
commit
375a341aac
@ -17,8 +17,7 @@
|
||||
"server/"
|
||||
],
|
||||
"scripts": {
|
||||
"lint:check": "eslint . && yarn prettier --check .",
|
||||
"lint:fix": "eslint . --fix && yarn prettier --write .",
|
||||
"build": "tsc --build",
|
||||
"test": "jest --forceExit --detectOpenHandles --runInBand",
|
||||
"test:watch": "jest --forceExit --detectOpenHandles --runInBand --watch",
|
||||
"release": "standard-version",
|
||||
@ -54,7 +53,8 @@
|
||||
"prettier": "^2.7.1",
|
||||
"sqlite3": "^5.0.11",
|
||||
"standard-version": "^9.5.0",
|
||||
"supertest": "^6.2.4"
|
||||
"supertest": "^6.2.4",
|
||||
"typescript": "^5.0.3"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@strapi/strapi": "^4.0.0"
|
||||
|
20
src/libs/arrays.ts
Normal file
20
src/libs/arrays.ts
Normal file
@ -0,0 +1,20 @@
|
||||
/**
|
||||
* Check if value is an array.
|
||||
*/
|
||||
const isArraySafe = <T>(val: T | T[]): val is T[] => {
|
||||
return val && Array.isArray(val);
|
||||
};
|
||||
|
||||
/**
|
||||
* Convert value to array if not already.
|
||||
* @param {*} val
|
||||
* @returns {Array<*>}
|
||||
*/
|
||||
const toArray = <T>(val: T | T[]): T[] => {
|
||||
return isArraySafe(val) ? val : [val];
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
isArraySafe,
|
||||
toArray,
|
||||
};
|
52
src/libs/objects.js
Normal file
52
src/libs/objects.js
Normal file
@ -0,0 +1,52 @@
|
||||
const deepmerge = require('deepmerge');
|
||||
|
||||
class ObjectBuilder {
|
||||
_obj = {};
|
||||
|
||||
get() {
|
||||
return this._obj;
|
||||
}
|
||||
|
||||
extend(obj) {
|
||||
this._obj = { ...this._obj, ...obj };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if value is an object.
|
||||
* @param {*} val
|
||||
* @returns {boolean}
|
||||
*/
|
||||
const isObjectSafe = (val) => {
|
||||
return val && !Array.isArray(val) && typeof val === 'object';
|
||||
};
|
||||
|
||||
const isObjectEmpty = (obj) => {
|
||||
for (let i in obj) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
const mergeObjects = (x, y) => {
|
||||
return deepmerge(x, y, {
|
||||
arrayMerge: (target, source) => {
|
||||
source.forEach((item) => {
|
||||
if (target.indexOf(item) === -1) {
|
||||
target.push(item);
|
||||
}
|
||||
});
|
||||
return target;
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const logObj = (obj) => JSON.stringify(obj, null, ' ');
|
||||
|
||||
module.exports = {
|
||||
ObjectBuilder,
|
||||
logObj,
|
||||
isObjectSafe,
|
||||
isObjectEmpty,
|
||||
mergeObjects,
|
||||
};
|
5
src/server/bootstrap.js
vendored
Normal file
5
src/server/bootstrap.js
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = ({ strapi }) => {
|
||||
// bootstrap phase
|
||||
};
|
18
src/server/config/constants.js
Normal file
18
src/server/config/constants.js
Normal file
@ -0,0 +1,18 @@
|
||||
const CustomSlugs = {
|
||||
MEDIA: 'media',
|
||||
WHOLE_DB: 'custom:db',
|
||||
};
|
||||
|
||||
const CustomSlugToSlug = {
|
||||
[CustomSlugs.MEDIA]: 'plugin::upload.file',
|
||||
};
|
||||
|
||||
const isCustomSlug = (slug) => {
|
||||
return !!CustomSlugToSlug[slug];
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
CustomSlugs,
|
||||
CustomSlugToSlug,
|
||||
isCustomSlug,
|
||||
};
|
15
src/server/config/index.js
Normal file
15
src/server/config/index.js
Normal file
@ -0,0 +1,15 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = {
|
||||
default: {
|
||||
/**
|
||||
* Public hostname of the server.
|
||||
*/
|
||||
serverPublicHostname: '',
|
||||
},
|
||||
validator: ({ serverPublicHostname } = {}) => {
|
||||
if (typeof serverPublicHostname !== 'string') {
|
||||
throw new Error('serverPublicHostname has to be a string.');
|
||||
}
|
||||
},
|
||||
};
|
3
src/server/content-types/index.js
Normal file
3
src/server/content-types/index.js
Normal file
@ -0,0 +1,3 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = {};
|
@ -0,0 +1,43 @@
|
||||
'use strict';
|
||||
|
||||
const { CustomSlugs } = require('../../../config/constants');
|
||||
const { getService } = require('../../../utils');
|
||||
const { getAllSlugs } = require('../../../utils/models');
|
||||
const { handleAsyncError } = require('../../content-api/utils');
|
||||
|
||||
const exportData = async (ctx) => {
|
||||
if (!hasPermissions(ctx)) {
|
||||
return ctx.forbidden();
|
||||
}
|
||||
|
||||
let { slug, search, applySearch, exportFormat, relationsAsId, deepness = 5 } = ctx.request.body;
|
||||
|
||||
let data;
|
||||
if (exportFormat === getService('export').formats.JSON_V2) {
|
||||
data = await getService('export').exportDataV2({ slug, search, applySearch, deepness });
|
||||
} else {
|
||||
data = await getService('export').exportData({ slug, search, applySearch, exportFormat, relationsAsId, deepness });
|
||||
}
|
||||
|
||||
ctx.body = {
|
||||
data,
|
||||
};
|
||||
};
|
||||
|
||||
const hasPermissions = (ctx) => {
|
||||
let { slug } = ctx.request.body;
|
||||
const { userAbility } = ctx.state;
|
||||
|
||||
const slugs = slug === CustomSlugs.WHOLE_DB ? getAllSlugs() : [slug];
|
||||
|
||||
const allowedSlugs = slugs.filter((slug) => {
|
||||
const permissionChecker = strapi.plugin('content-manager').service('permission-checker').create({ userAbility, model: slug });
|
||||
return permissionChecker.can.read();
|
||||
});
|
||||
|
||||
return !!allowedSlugs.length;
|
||||
};
|
||||
|
||||
module.exports = ({ strapi }) => ({
|
||||
exportData: handleAsyncError(exportData),
|
||||
});
|
1
src/server/controllers/admin/export-controller/index.js
Normal file
1
src/server/controllers/admin/export-controller/index.js
Normal file
@ -0,0 +1 @@
|
||||
module.exports = require('./export-controller');
|
@ -0,0 +1,28 @@
|
||||
'use strict';
|
||||
|
||||
const { getModelAttributes } = require('../../../utils/models');
|
||||
|
||||
const getModelAttributesEndpoint = async (ctx) => {
|
||||
const { slug } = ctx.params;
|
||||
|
||||
const attributeNames = getModelAttributes(slug)
|
||||
.filter(filterAttribute)
|
||||
.map((attr) => attr.name);
|
||||
|
||||
ctx.body = {
|
||||
data: {
|
||||
attribute_names: attributeNames,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const filterAttribute = (attr) => {
|
||||
const filters = [filterType, filterName];
|
||||
return filters.every((filter) => filter(attr));
|
||||
};
|
||||
|
||||
const filterType = (attr) => !['relation', 'component', 'dynamiczone'].includes(attr.type);
|
||||
|
||||
const filterName = (attr) => !['createdAt', 'updatedAt', 'publishedAt', 'locale'].includes(attr.name);
|
||||
|
||||
module.exports = ({ strapi }) => getModelAttributesEndpoint;
|
@ -0,0 +1,9 @@
|
||||
'use strict';
|
||||
|
||||
const getModelAttributes = require('./get-model-attributes');
|
||||
const importData = require('./import-data');
|
||||
|
||||
module.exports = ({ strapi }) => ({
|
||||
getModelAttributes: getModelAttributes({ strapi }),
|
||||
importData: importData({ strapi }),
|
||||
});
|
@ -0,0 +1,45 @@
|
||||
'use strict';
|
||||
|
||||
const { getService } = require('../../../utils');
|
||||
|
||||
const importData = async (ctx) => {
|
||||
if (!hasPermissions(ctx)) {
|
||||
return ctx.forbidden();
|
||||
}
|
||||
|
||||
const { user } = ctx.state;
|
||||
const { slug, data: dataRaw, format, idField } = ctx.request.body;
|
||||
|
||||
const fileContent = await getService('import').parseInputData(format, dataRaw, { slug });
|
||||
|
||||
let res;
|
||||
if (fileContent?.version === 2) {
|
||||
res = await getService('import').importDataV2(fileContent, {
|
||||
slug,
|
||||
user,
|
||||
idField,
|
||||
});
|
||||
} else {
|
||||
res = await getService('import').importData(dataRaw, {
|
||||
slug,
|
||||
format,
|
||||
user,
|
||||
idField,
|
||||
});
|
||||
}
|
||||
|
||||
ctx.body = {
|
||||
failures: res.failures,
|
||||
};
|
||||
};
|
||||
|
||||
const hasPermissions = (ctx) => {
|
||||
let { slug } = ctx.request.body;
|
||||
const { userAbility } = ctx.state;
|
||||
|
||||
const permissionChecker = strapi.plugin('content-manager').service('permission-checker').create({ userAbility, model: slug });
|
||||
|
||||
return permissionChecker.can.create() && permissionChecker.can.update();
|
||||
};
|
||||
|
||||
module.exports = ({ strapi }) => importData;
|
1
src/server/controllers/admin/import-controller/index.js
Normal file
1
src/server/controllers/admin/import-controller/index.js
Normal file
@ -0,0 +1 @@
|
||||
module.exports = require('./import-controller');
|
@ -0,0 +1,34 @@
|
||||
'use strict';
|
||||
|
||||
const Joi = require('joi');
|
||||
|
||||
const { getService } = require('../../../utils');
|
||||
const { checkParams, handleAsyncError } = require('../utils');
|
||||
|
||||
const bodySchema = Joi.object({
|
||||
slug: Joi.string().required(),
|
||||
exportFormat: Joi.string().valid('csv', 'json', 'json-v2').required(),
|
||||
search: Joi.string().default(''),
|
||||
applySearch: Joi.boolean().default(false),
|
||||
relationsAsId: Joi.boolean().default(false),
|
||||
deepness: Joi.number().integer().min(1).default(5),
|
||||
});
|
||||
|
||||
const exportData = async (ctx) => {
|
||||
let { slug, search, applySearch, exportFormat, relationsAsId, deepness } = checkParams(bodySchema, ctx.request.body);
|
||||
|
||||
let data;
|
||||
if (exportFormat === getService('export').formats.JSON_V2) {
|
||||
data = await getService('export').exportDataV2({ slug, search, applySearch, deepness });
|
||||
} else {
|
||||
data = await getService('export').exportData({ slug, search, applySearch, exportFormat, relationsAsId, deepness });
|
||||
}
|
||||
|
||||
ctx.body = {
|
||||
data,
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = ({ strapi }) => ({
|
||||
exportData: handleAsyncError(exportData),
|
||||
});
|
@ -0,0 +1 @@
|
||||
module.exports = require('./export-controller');
|
@ -0,0 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const importData = require('./import-data');
|
||||
|
||||
module.exports = ({ strapi }) => ({
|
||||
importData: importData({ strapi }),
|
||||
});
|
@ -0,0 +1,43 @@
|
||||
'use strict';
|
||||
|
||||
const Joi = require('joi');
|
||||
|
||||
const { getService } = require('../../../utils');
|
||||
const { checkParams, handleAsyncError } = require('../utils');
|
||||
|
||||
const bodySchema = Joi.object({
|
||||
slug: Joi.string().required(),
|
||||
data: Joi.any().required(),
|
||||
format: Joi.string().valid('csv', 'json').required(),
|
||||
idField: Joi.string(),
|
||||
});
|
||||
|
||||
const importData = async (ctx) => {
|
||||
const { user } = ctx.state;
|
||||
|
||||
const { slug, data: dataRaw, format, idField } = checkParams(bodySchema, ctx.request.body);
|
||||
|
||||
const fileContent = await getService('import').parseInputData(format, dataRaw, { slug });
|
||||
|
||||
let res;
|
||||
if (fileContent?.version === 2) {
|
||||
res = await getService('import').importDataV2(fileContent, {
|
||||
slug,
|
||||
user,
|
||||
idField,
|
||||
});
|
||||
} else {
|
||||
res = await getService('import').importData(dataRaw, {
|
||||
slug,
|
||||
format,
|
||||
user,
|
||||
idField,
|
||||
});
|
||||
}
|
||||
|
||||
ctx.body = {
|
||||
failures: res.failures,
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = ({ strapi }) => handleAsyncError(importData);
|
@ -0,0 +1 @@
|
||||
module.exports = require('./import-controller');
|
49
src/server/controllers/content-api/utils.js
Normal file
49
src/server/controllers/content-api/utils.js
Normal file
@ -0,0 +1,49 @@
|
||||
const { BusinessError } = require('../../utils/errors');
|
||||
|
||||
/**
|
||||
* Check an object's properties based on a validation schema
|
||||
* @param {*} schema - Validation schema
|
||||
* @param {*} obj - Object to check properties
|
||||
* @param {Object} options
|
||||
* @param {boolean} [options.allowUnknown] - Whether to allow unknown properties (default: false)
|
||||
* @returns Object with values checked and parsed
|
||||
*/
|
||||
const checkParams = (schema, obj, options = {}) => {
|
||||
const allowUnknown = options.allowUnknown || false;
|
||||
|
||||
const validation = schema.validate(obj, {
|
||||
abortEarly: false,
|
||||
allowUnknown,
|
||||
});
|
||||
|
||||
if (validation.error) {
|
||||
const error = validation.error.details.map((detail) => detail.message).join(', ');
|
||||
throw new BusinessError(error);
|
||||
}
|
||||
|
||||
return validation.value;
|
||||
};
|
||||
|
||||
const handleAsyncError = (fn) => async (ctx) => {
|
||||
try {
|
||||
const res = await fn(ctx);
|
||||
return res;
|
||||
} catch (err) {
|
||||
strapi.log.error(err);
|
||||
|
||||
if (err instanceof BusinessError) {
|
||||
ctx.status = 400;
|
||||
ctx.body = {
|
||||
message: err.message,
|
||||
code: err.code,
|
||||
};
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
checkParams,
|
||||
handleAsyncError,
|
||||
};
|
13
src/server/controllers/index.js
Normal file
13
src/server/controllers/index.js
Normal file
@ -0,0 +1,13 @@
|
||||
'use strict';
|
||||
|
||||
const exportAdminController = require('./admin/export-controller');
|
||||
const importAdminController = require('./admin/import-controller');
|
||||
const exportContentApiController = require('./content-api/export-controller');
|
||||
const importContentApiController = require('./content-api/import-controller');
|
||||
|
||||
module.exports = {
|
||||
exportAdmin: exportAdminController,
|
||||
importAdmin: importAdminController,
|
||||
export: exportContentApiController,
|
||||
import: importContentApiController,
|
||||
};
|
3
src/server/destroy.js
Normal file
3
src/server/destroy.js
Normal file
@ -0,0 +1,3 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = ({ strapi }) => {};
|
25
src/server/index.js
Normal file
25
src/server/index.js
Normal file
@ -0,0 +1,25 @@
|
||||
'use strict';
|
||||
|
||||
const register = require('./register');
|
||||
const bootstrap = require('./bootstrap');
|
||||
const destroy = require('./destroy');
|
||||
const config = require('./config');
|
||||
const contentTypes = require('./content-types');
|
||||
const controllers = require('./controllers');
|
||||
const routes = require('./routes');
|
||||
const middlewares = require('./middlewares');
|
||||
const policies = require('./policies');
|
||||
const services = require('./services');
|
||||
|
||||
module.exports = {
|
||||
register,
|
||||
bootstrap,
|
||||
destroy,
|
||||
config,
|
||||
controllers,
|
||||
routes,
|
||||
services,
|
||||
contentTypes,
|
||||
policies,
|
||||
middlewares,
|
||||
};
|
3
src/server/middlewares/index.js
Normal file
3
src/server/middlewares/index.js
Normal file
@ -0,0 +1,3 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = {};
|
3
src/server/policies/index.js
Normal file
3
src/server/policies/index.js
Normal file
@ -0,0 +1,3 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = {};
|
5
src/server/register.js
Normal file
5
src/server/register.js
Normal file
@ -0,0 +1,5 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = ({ strapi }) => {
|
||||
// registeration phase
|
||||
};
|
13
src/server/routes/export-admin.js
Normal file
13
src/server/routes/export-admin.js
Normal file
@ -0,0 +1,13 @@
|
||||
module.exports = {
|
||||
type: 'admin',
|
||||
routes: [
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/export/contentTypes',
|
||||
handler: 'exportAdmin.exportData',
|
||||
config: {
|
||||
policies: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
13
src/server/routes/export-content-api.js
Normal file
13
src/server/routes/export-content-api.js
Normal file
@ -0,0 +1,13 @@
|
||||
module.exports = {
|
||||
type: 'content-api',
|
||||
routes: [
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/content/export/contentTypes',
|
||||
handler: 'export.exportData',
|
||||
config: {
|
||||
policies: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
21
src/server/routes/import-admin.js
Normal file
21
src/server/routes/import-admin.js
Normal file
@ -0,0 +1,21 @@
|
||||
module.exports = {
|
||||
type: 'admin',
|
||||
routes: [
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/import/model-attributes/:slug',
|
||||
handler: 'importAdmin.getModelAttributes',
|
||||
config: {
|
||||
policies: [],
|
||||
},
|
||||
},
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/import',
|
||||
handler: 'importAdmin.importData',
|
||||
config: {
|
||||
policies: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
13
src/server/routes/import-content-api.js
Normal file
13
src/server/routes/import-content-api.js
Normal file
@ -0,0 +1,13 @@
|
||||
module.exports = {
|
||||
type: 'content-api',
|
||||
routes: [
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/content/import',
|
||||
handler: 'import.importData',
|
||||
config: {
|
||||
policies: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
11
src/server/routes/index.js
Normal file
11
src/server/routes/index.js
Normal file
@ -0,0 +1,11 @@
|
||||
const exportAdminRoutes = require('./export-admin');
|
||||
const importAdminRoutes = require('./import-admin');
|
||||
const exportContentApiRoutes = require('./export-content-api');
|
||||
const importContentApiRoutes = require('./import-content-api');
|
||||
|
||||
module.exports = {
|
||||
exportAdminRoutes,
|
||||
importAdminRoutes,
|
||||
export: exportContentApiRoutes,
|
||||
import: importContentApiRoutes,
|
||||
};
|
75
src/server/services/export/converters-v2.js
Normal file
75
src/server/services/export/converters-v2.js
Normal file
@ -0,0 +1,75 @@
|
||||
const fromPairs = require('lodash/fromPairs');
|
||||
const pick = require('lodash/pick');
|
||||
const toPairs = require('lodash/toPairs');
|
||||
const { CustomSlugToSlug, CustomSlugs } = require('../../config/constants');
|
||||
const { getConfig } = require('../../utils/getConfig');
|
||||
|
||||
const convertToJson = (jsoContent) => {
|
||||
return JSON.stringify(jsoContent, null, '\t');
|
||||
};
|
||||
|
||||
const withBeforeConvert = (convertFn) => (jsoContent, options) => {
|
||||
jsoContent = beforeConvert(jsoContent, options);
|
||||
jsoContent = convertFn(jsoContent, options);
|
||||
return jsoContent;
|
||||
};
|
||||
|
||||
const beforeConvert = (jsoContent, options) => {
|
||||
jsoContent = preprocess(jsoContent, options);
|
||||
jsoContent = postprocess(jsoContent, options);
|
||||
|
||||
return jsoContent;
|
||||
};
|
||||
|
||||
const preprocess = (jsoContent) => {
|
||||
let media = jsoContent.data[CustomSlugToSlug[CustomSlugs.MEDIA]];
|
||||
|
||||
if (!media) {
|
||||
return jsoContent;
|
||||
}
|
||||
|
||||
media = fromPairs(
|
||||
toPairs(media).map(([id, medium]) => {
|
||||
if (isRelativeUrl(medium.url)) {
|
||||
medium.url = buildAbsoluteUrl(medium.url);
|
||||
}
|
||||
return [id, medium];
|
||||
}),
|
||||
);
|
||||
|
||||
jsoContent.data[CustomSlugToSlug[CustomSlugs.MEDIA]] = media;
|
||||
|
||||
return jsoContent;
|
||||
};
|
||||
|
||||
const isRelativeUrl = (url) => {
|
||||
return url.startsWith('/');
|
||||
};
|
||||
|
||||
const buildAbsoluteUrl = (relativeUrl) => {
|
||||
return getConfig('serverPublicHostname') + relativeUrl;
|
||||
};
|
||||
|
||||
const postprocess = (jsoContent) => {
|
||||
let mediaSlug = CustomSlugToSlug[CustomSlugs.MEDIA];
|
||||
let media = jsoContent.data[mediaSlug];
|
||||
|
||||
if (!media) {
|
||||
return jsoContent;
|
||||
}
|
||||
|
||||
media = fromPairs(
|
||||
toPairs(media).map(([id, medium]) => {
|
||||
medium = pick(medium, ['id', 'name', 'alternativeText', 'caption', 'hash', 'ext', 'mime', 'url', 'createdAt', 'updatedAt', 'createdBy', 'updatedBy']);
|
||||
return [id, medium];
|
||||
}),
|
||||
);
|
||||
|
||||
jsoContent.data[mediaSlug] = media;
|
||||
|
||||
return jsoContent;
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
convertToJson: withBeforeConvert(convertToJson),
|
||||
};
|
145
src/server/services/export/converters.js
Normal file
145
src/server/services/export/converters.js
Normal file
@ -0,0 +1,145 @@
|
||||
const { isArraySafe, toArray } = require('../../../libs/arrays');
|
||||
const { isObjectSafe } = require('../../../libs/objects');
|
||||
const { CustomSlugToSlug, CustomSlugs } = require('../../config/constants');
|
||||
const { getConfig } = require('../../utils/getConfig');
|
||||
const { getModelAttributes, getModel } = require('../../utils/models');
|
||||
|
||||
const convertToCsv = (entries, options) => {
|
||||
entries = toArray(entries);
|
||||
const columnTitles = ['id'].concat(getModelAttributes(options.slug, { filterOutTarget: ['admin::user'] }).map((attr) => attr.name));
|
||||
const content = [convertStrArrayToCsv(columnTitles)].concat(entries.map((entry) => convertEntryToStrArray(entry, columnTitles)).map(convertStrArrayToCsv)).join('\r\n');
|
||||
return content;
|
||||
};
|
||||
|
||||
const convertStrArrayToCsv = (entry) => {
|
||||
return entry
|
||||
.map(stringifyEntry)
|
||||
.map((v) => v.replace(/"/g, '""'))
|
||||
.map((v) => `"${v}"`)
|
||||
.join(',');
|
||||
};
|
||||
|
||||
const stringifyEntry = (entry) => {
|
||||
if (typeof entry === 'object') {
|
||||
return JSON.stringify(entry);
|
||||
}
|
||||
|
||||
return String(entry);
|
||||
};
|
||||
|
||||
const convertEntryToStrArray = (entry, keys) => {
|
||||
return keys.map((key) => entry[key]);
|
||||
};
|
||||
|
||||
const convertToJson = (entries, options) => {
|
||||
entries = JSON.stringify(entries, null, '\t');
|
||||
return entries;
|
||||
};
|
||||
|
||||
const withBeforeConvert = (convertFn) => (entries, options) => {
|
||||
entries = beforeConvert(entries, options);
|
||||
entries = convertFn(entries, options);
|
||||
return entries;
|
||||
};
|
||||
|
||||
const beforeConvert = (entries, options) => {
|
||||
entries = toArray(entries);
|
||||
|
||||
entries = exportMedia(entries, options);
|
||||
if (options.relationsAsId) {
|
||||
entries = exportRelationsAsId(entries, options);
|
||||
}
|
||||
|
||||
if (getModel(options.slug).kind === 'singleType') {
|
||||
return entries?.[0];
|
||||
}
|
||||
return entries;
|
||||
};
|
||||
|
||||
const exportMedia = (entries, options) => {
|
||||
if (options.slug === CustomSlugToSlug[CustomSlugs.MEDIA]) {
|
||||
entries = entries.map((entry) => {
|
||||
if (isObjectSafe(entry) && entry.url.startsWith('/')) {
|
||||
entry.url = computeUrl(entry.url);
|
||||
}
|
||||
return entry;
|
||||
});
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
const mediaKeys = getModelAttributes(options.slug, { filterOutTarget: ['admin::user'], filterType: ['media'] }).map((attr) => attr.name);
|
||||
const relationsAttr = getModelAttributes(options.slug, { filterOutTarget: ['admin::user'], filterType: ['component', 'dynamiczone', 'relation'] });
|
||||
|
||||
entries = entries.map((entry) => {
|
||||
mediaKeys.forEach((key) => {
|
||||
if (isArraySafe(entry[key])) {
|
||||
entry[key] = entry[key].map((entryItem) => {
|
||||
if (isObjectSafe(entryItem) && entryItem.url.startsWith('/')) {
|
||||
entryItem.url = computeUrl(entryItem.url);
|
||||
}
|
||||
return entryItem;
|
||||
});
|
||||
} else if (isObjectSafe(entry[key]) && entry[key].url.startsWith('/')) {
|
||||
entry[key].url = computeUrl(entry[key].url);
|
||||
}
|
||||
});
|
||||
|
||||
relationsAttr.forEach((attr) => {
|
||||
if (attr.type === 'component') {
|
||||
if (entry[attr.name]) {
|
||||
const areMultiple = attr.repeatable;
|
||||
const relEntriesProcessed = exportMedia(toArray(entry[attr.name]), { slug: attr.component });
|
||||
entry[attr.name] = areMultiple ? relEntriesProcessed : relEntriesProcessed?.[0] || null;
|
||||
}
|
||||
} else if (attr.type === 'dynamiczone') {
|
||||
if (entry[attr.name]) {
|
||||
entry[attr.name] = entry[attr.name].map((component) => exportMedia([component], { slug: component.__component })?.[0] || null);
|
||||
}
|
||||
} else if (attr.type === 'relation') {
|
||||
if (entry[attr.name]) {
|
||||
const areMultiple = isArraySafe(entry[attr.name]);
|
||||
const relEntriesProcessed = exportMedia(toArray(entry[attr.name]), { slug: attr.target });
|
||||
entry[attr.name] = areMultiple ? relEntriesProcessed : relEntriesProcessed?.[0] || null;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return entry;
|
||||
});
|
||||
|
||||
return entries;
|
||||
};
|
||||
|
||||
const computeUrl = (relativeUrl) => {
|
||||
return getConfig('serverPublicHostname') + relativeUrl;
|
||||
};
|
||||
|
||||
const exportRelationsAsId = (entries, options) => {
|
||||
const relationKeys = getModelAttributes(options.slug, { filterOutTarget: ['admin::user'], filterType: ['component', 'dynamiczone', 'media', 'relation'] }).map(
|
||||
(attr) => attr.name,
|
||||
);
|
||||
|
||||
return entries.map((entry) => {
|
||||
relationKeys.forEach((key) => {
|
||||
if (entry[key] == null) {
|
||||
entry[key] = null;
|
||||
} else if (isArraySafe(entry[key])) {
|
||||
entry[key] = entry[key].map((rel) => {
|
||||
if (typeof rel === 'object') {
|
||||
return rel.id;
|
||||
}
|
||||
return rel;
|
||||
});
|
||||
} else if (isObjectSafe(entry[key])) {
|
||||
entry[key] = entry[key].id;
|
||||
}
|
||||
});
|
||||
return entry;
|
||||
});
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
convertToCsv: withBeforeConvert(convertToCsv),
|
||||
convertToJson: withBeforeConvert(convertToJson),
|
||||
};
|
390
src/server/services/export/export-v2.js
Normal file
390
src/server/services/export/export-v2.js
Normal file
@ -0,0 +1,390 @@
|
||||
const cloneDeep = require('lodash/cloneDeep');
|
||||
const flattenDeep = require('lodash/flattenDeep');
|
||||
const fromPairs = require('lodash/fromPairs');
|
||||
const { isEmpty, merge } = require('lodash/fp');
|
||||
const qs = require('qs');
|
||||
const { isArraySafe, toArray } = require('../../../libs/arrays');
|
||||
|
||||
const { ObjectBuilder, isObjectSafe, mergeObjects } = require('../../../libs/objects');
|
||||
const { CustomSlugToSlug, CustomSlugs } = require('../../config/constants');
|
||||
const { getModelAttributes, getAllSlugs } = require('../../utils/models');
|
||||
const { convertToJson } = require('./converters-v2');
|
||||
|
||||
const dataFormats = {
|
||||
JSON: 'json',
|
||||
};
|
||||
|
||||
const dataConverterConfigs = {
|
||||
[dataFormats.JSON]: {
|
||||
convertEntries: convertToJson,
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Export data.
|
||||
* @param {Object} options
|
||||
* @param {string} options.slug
|
||||
* @param {string} options.search
|
||||
* @param {boolean} options.applySearch
|
||||
* @param {boolean} options.relationsAsId
|
||||
* @param {number} options.deepness
|
||||
* @returns {string}
|
||||
*/
|
||||
const exportDataV2 = async ({ slug, search, applySearch, deepness = 5 }) => {
|
||||
slug = CustomSlugToSlug[slug] || slug;
|
||||
|
||||
let entries = {};
|
||||
if (slug === CustomSlugs.WHOLE_DB) {
|
||||
for (const slug of getAllSlugs()) {
|
||||
const hierarchy = buildSlugHierarchy(slug, deepness);
|
||||
const slugEntries = await findEntriesForHierarchy(slug, hierarchy, deepness, { ...(applySearch ? { search } : {}) });
|
||||
entries = mergeObjects(entries, slugEntries);
|
||||
}
|
||||
} else {
|
||||
const hierarchy = buildSlugHierarchy(slug, deepness);
|
||||
entries = await findEntriesForHierarchy(slug, hierarchy, deepness, { ...(applySearch ? { search } : {}) });
|
||||
}
|
||||
|
||||
const jsoContent = {
|
||||
version: 2,
|
||||
data: entries,
|
||||
};
|
||||
|
||||
const fileContent = convertData(jsoContent, {
|
||||
slug,
|
||||
dataFormat: 'json',
|
||||
});
|
||||
|
||||
return fileContent;
|
||||
};
|
||||
|
||||
const findEntriesForHierarchy = async (slug, hierarchy, deepness, { search, ids }) => {
|
||||
let storedData = {};
|
||||
|
||||
if (slug === 'admin::user') {
|
||||
return storedData;
|
||||
}
|
||||
|
||||
let entries = await findEntries(slug, deepness, { search, ids })
|
||||
.then((entries) => {
|
||||
entries = toArray(entries).filter(Boolean);
|
||||
const isModelLocalized = !!hierarchy?.localizations;
|
||||
|
||||
// Export locales
|
||||
if (isModelLocalized) {
|
||||
const allEntries = [...entries];
|
||||
const entryIdsToExported = fromPairs(allEntries.map((entry) => [entry.id, true]));
|
||||
|
||||
for (const entry of entries) {
|
||||
entry.localizations.forEach((locale) => {
|
||||
if (!entryIdsToExported[locale.id]) {
|
||||
allEntries.push(locale);
|
||||
entryIdsToExported[locale.id] = true;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return allEntries;
|
||||
}
|
||||
|
||||
return entries;
|
||||
})
|
||||
.then((entries) => toArray(entries));
|
||||
|
||||
// Transform relations as ids.
|
||||
let entriesFlatten = cloneDeep(entries);
|
||||
(() => {
|
||||
const flattenEntryDynamicZone = (dynamicZoneEntries) => {
|
||||
if (isArraySafe(dynamicZoneEntries)) {
|
||||
return dynamicZoneEntries.map((entry) => ({
|
||||
__component: entry.__component,
|
||||
id: entry.id,
|
||||
}));
|
||||
}
|
||||
return dynamicZoneEntries;
|
||||
};
|
||||
|
||||
const flattenEntryCommon = (entry) => {
|
||||
if (entry == null) {
|
||||
return null;
|
||||
} else if (isArraySafe(entry)) {
|
||||
return entry.map((rel) => {
|
||||
if (typeof rel === 'object') {
|
||||
return rel.id;
|
||||
}
|
||||
return rel;
|
||||
});
|
||||
} else if (isObjectSafe(entry)) {
|
||||
return entry.id;
|
||||
}
|
||||
return entry;
|
||||
};
|
||||
|
||||
const flattenEntry = (entry, slug) => {
|
||||
const attributes = getModelAttributes(slug, { filterType: ['component', 'dynamiczone', 'media', 'relation'] });
|
||||
|
||||
attributes.forEach((attribute) => {
|
||||
const flattener = attribute.type === 'dynamiczone' ? flattenEntryDynamicZone : flattenEntryCommon;
|
||||
entry[attribute.name] = flattener(entry[attribute.name]);
|
||||
});
|
||||
|
||||
return entry;
|
||||
};
|
||||
|
||||
entriesFlatten = entriesFlatten.map((entry) => flattenEntry(entry, slug));
|
||||
})();
|
||||
|
||||
storedData = mergeObjects({ [slug]: Object.fromEntries(entriesFlatten.map((entry) => [entry.id, entry])) }, storedData);
|
||||
|
||||
// Skip admin::user slug.
|
||||
(() => {
|
||||
const relations = getModelAttributes(slug, { filterType: ['relation'] });
|
||||
|
||||
return entries.map((entry) => {
|
||||
relations.forEach((relation) => {
|
||||
if (relation.target === 'admin::user') {
|
||||
delete entry[relation.name];
|
||||
}
|
||||
});
|
||||
return entry;
|
||||
});
|
||||
})();
|
||||
|
||||
await (async () => {
|
||||
let attributes = getModelAttributes(slug, { filterType: ['component'] });
|
||||
for (const attribute of attributes) {
|
||||
if (!hierarchy[attribute.name]?.__slug) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const ids = flattenDeep(
|
||||
entries
|
||||
.filter((entry) => !!entry[attribute.name])
|
||||
.map((entry) => entry[attribute.name])
|
||||
.map(toArray)
|
||||
.map((entryArray) => entryArray.filter((entry) => !!entry.id).map((entry) => entry.id)),
|
||||
);
|
||||
|
||||
const subStore = await findEntriesForHierarchy(hierarchy[attribute.name].__slug, hierarchy[attribute.name], deepness - 1, { ids });
|
||||
storedData = mergeObjects(subStore, storedData);
|
||||
}
|
||||
})();
|
||||
|
||||
await (async () => {
|
||||
let attributes = getModelAttributes(slug, { filterType: ['dynamiczone'] });
|
||||
for (const attribute of attributes) {
|
||||
for (const componentSlug of attribute.components) {
|
||||
const componentHierarchy = hierarchy[attribute.name]?.[componentSlug];
|
||||
if (!componentHierarchy?.__slug) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const componentEntries = flattenDeep(
|
||||
entries
|
||||
.map((entry) => {
|
||||
return entry;
|
||||
})
|
||||
.filter((entry) => !!entry[attribute.name])
|
||||
.map((entry) => entry[attribute.name]),
|
||||
).filter((entry) => entry?.__component === componentSlug);
|
||||
|
||||
const ids = componentEntries.map((entry) => entry.id);
|
||||
|
||||
const subStore = await findEntriesForHierarchy(componentHierarchy.__slug, componentHierarchy, deepness - 1, { ids });
|
||||
storedData = mergeObjects(subStore, storedData);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
await (async () => {
|
||||
let attributes = getModelAttributes(slug, { filterType: ['media'] });
|
||||
for (const attribute of attributes) {
|
||||
if (!hierarchy[attribute.name]?.__slug) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const ids = flattenDeep(
|
||||
entries
|
||||
.filter((entry) => !!entry[attribute.name])
|
||||
.map((entry) => entry[attribute.name])
|
||||
.map(toArray)
|
||||
.map((entryArray) => entryArray.filter((entry) => !!entry.id).map((entry) => entry.id)),
|
||||
);
|
||||
|
||||
const subStore = await findEntriesForHierarchy(hierarchy[attribute.name].__slug, hierarchy[attribute.name], deepness - 1, { ids });
|
||||
storedData = mergeObjects(subStore, storedData);
|
||||
}
|
||||
})();
|
||||
|
||||
await (async () => {
|
||||
let attributes = getModelAttributes(slug, { filterType: ['relation'] });
|
||||
for (const attribute of attributes) {
|
||||
if (!hierarchy[attribute.name]?.__slug) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const ids = flattenDeep(
|
||||
entries
|
||||
.filter((entry) => !!entry[attribute.name])
|
||||
.map((entry) => entry[attribute.name])
|
||||
.map(toArray)
|
||||
.map((entryArray) => entryArray.filter((entry) => !!entry.id).map((entry) => entry.id)),
|
||||
);
|
||||
|
||||
const subStore = await findEntriesForHierarchy(hierarchy[attribute.name].__slug, hierarchy[attribute.name], deepness - 1, { ids });
|
||||
storedData = mergeObjects(subStore, storedData);
|
||||
}
|
||||
})();
|
||||
|
||||
return storedData;
|
||||
};
|
||||
|
||||
const findEntries = async (slug, deepness, { search, ids }) => {
|
||||
try {
|
||||
const queryBuilder = new ObjectBuilder();
|
||||
queryBuilder.extend(getPopulateFromSchema(slug, deepness));
|
||||
if (search) {
|
||||
queryBuilder.extend(buildFilterQuery(search));
|
||||
} else if (ids) {
|
||||
queryBuilder.extend({
|
||||
filters: {
|
||||
id: { $in: ids },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const entries = await strapi.entityService.findMany(slug, queryBuilder.get());
|
||||
|
||||
return entries;
|
||||
} catch (_) {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
const buildFilterQuery = (search) => {
|
||||
let { filters, sort: sortRaw } = qs.parse(search);
|
||||
|
||||
const [attr, value] = sortRaw?.split(':') || [];
|
||||
let sort = {};
|
||||
if (attr && value) {
|
||||
sort[attr] = value.toLowerCase();
|
||||
}
|
||||
|
||||
return {
|
||||
filters,
|
||||
sort,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Object} data
|
||||
* @param {Array<Object>} data.entries
|
||||
* @param {Record<string, any>} data.hierarchy
|
||||
* @param {Object} options
|
||||
* @param {string} options.slug
|
||||
* @param {string} options.dataFormat
|
||||
* @param {boolean} options.relationsAsId
|
||||
* @returns
|
||||
*/
|
||||
const convertData = (data, options) => {
|
||||
const converter = getConverter(options.dataFormat);
|
||||
|
||||
const convertedData = converter.convertEntries(data, options);
|
||||
|
||||
return convertedData;
|
||||
};
|
||||
|
||||
const getConverter = (dataFormat) => {
|
||||
const converter = dataConverterConfigs[dataFormat];
|
||||
|
||||
if (!converter) {
|
||||
throw new Error(`Data format ${dataFormat} is not supported.`);
|
||||
}
|
||||
|
||||
return converter;
|
||||
};
|
||||
|
||||
const getPopulateFromSchema = (slug, deepness = 5) => {
|
||||
if (deepness <= 1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (slug === 'admin::user') {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const populate = {};
|
||||
const model = strapi.getModel(slug);
|
||||
for (const [attributeName, attribute] of Object.entries(getModelPopulationAttributes(model))) {
|
||||
if (!attribute) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attribute.type === 'component') {
|
||||
populate[attributeName] = getPopulateFromSchema(attribute.component, deepness - 1);
|
||||
} else if (attribute.type === 'dynamiczone') {
|
||||
const dynamicPopulate = attribute.components.reduce((zonePopulate, component) => {
|
||||
const compPopulate = getPopulateFromSchema(component, deepness - 1);
|
||||
return compPopulate === true ? zonePopulate : merge(zonePopulate, compPopulate);
|
||||
}, {});
|
||||
populate[attributeName] = isEmpty(dynamicPopulate) ? true : dynamicPopulate;
|
||||
} else if (attribute.type === 'relation') {
|
||||
const relationPopulate = getPopulateFromSchema(attribute.target, deepness - 1);
|
||||
if (relationPopulate) {
|
||||
populate[attributeName] = relationPopulate;
|
||||
}
|
||||
} else if (attribute.type === 'media') {
|
||||
populate[attributeName] = true;
|
||||
}
|
||||
}
|
||||
|
||||
return isEmpty(populate) ? true : { populate };
|
||||
};
|
||||
|
||||
const buildSlugHierarchy = (slug, deepness = 5) => {
|
||||
slug = CustomSlugToSlug[slug] || slug;
|
||||
|
||||
if (deepness <= 1) {
|
||||
return { __slug: slug };
|
||||
}
|
||||
|
||||
const hierarchy = {
|
||||
__slug: slug,
|
||||
};
|
||||
|
||||
const model = strapi.getModel(slug);
|
||||
for (const [attributeName, attribute] of Object.entries(getModelPopulationAttributes(model))) {
|
||||
if (!attribute) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attribute.type === 'component') {
|
||||
hierarchy[attributeName] = buildSlugHierarchy(attribute.component, deepness - 1);
|
||||
} else if (attribute.type === 'dynamiczone') {
|
||||
hierarchy[attributeName] = Object.fromEntries(attribute.components.map((componentSlug) => [componentSlug, buildSlugHierarchy(componentSlug, deepness - 1)]));
|
||||
} else if (attribute.type === 'relation') {
|
||||
const relationHierarchy = buildSlugHierarchy(attribute.target, deepness - 1);
|
||||
if (relationHierarchy) {
|
||||
hierarchy[attributeName] = relationHierarchy;
|
||||
}
|
||||
} else if (attribute.type === 'media') {
|
||||
hierarchy[attributeName] = buildSlugHierarchy(CustomSlugs.MEDIA, deepness - 1);
|
||||
}
|
||||
}
|
||||
|
||||
return hierarchy;
|
||||
};
|
||||
|
||||
const getModelPopulationAttributes = (model) => {
|
||||
if (model.uid === 'plugin::upload.file') {
|
||||
const { related, ...attributes } = model.attributes;
|
||||
return attributes;
|
||||
}
|
||||
|
||||
return model.attributes;
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
exportDataV2,
|
||||
};
|
147
src/server/services/export/export.js
Normal file
147
src/server/services/export/export.js
Normal file
@ -0,0 +1,147 @@
|
||||
const { isEmpty, merge } = require('lodash/fp');
|
||||
const qs = require('qs');
|
||||
|
||||
const { ObjectBuilder } = require('../../../libs/objects');
|
||||
const { CustomSlugToSlug } = require('../../config/constants');
|
||||
const { convertToCsv, convertToJson } = require('./converters');
|
||||
|
||||
const dataFormats = {
|
||||
CSV: 'csv',
|
||||
JSON: 'json',
|
||||
JSON_V2: 'json-v2',
|
||||
};
|
||||
|
||||
const dataConverterConfigs = {
|
||||
[dataFormats.CSV]: {
|
||||
convertEntries: convertToCsv,
|
||||
},
|
||||
[dataFormats.JSON]: {
|
||||
convertEntries: convertToJson,
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Export data.
|
||||
* @param {Object} options
|
||||
* @param {string} options.slug
|
||||
* @param {("csv"|"json")} options.exportFormat
|
||||
* @param {string} options.search
|
||||
* @param {boolean} options.applySearch
|
||||
* @param {boolean} options.relationsAsId
|
||||
* @param {number} options.deepness
|
||||
* @returns {string}
|
||||
*/
|
||||
const exportData = async ({ slug, search, applySearch, exportFormat, relationsAsId, deepness = 5 }) => {
|
||||
const slugToProcess = CustomSlugToSlug[slug] || slug;
|
||||
|
||||
const queryBuilder = new ObjectBuilder();
|
||||
queryBuilder.extend(getPopulateFromSchema(slugToProcess, deepness));
|
||||
if (applySearch) {
|
||||
queryBuilder.extend(buildFilterQuery(search));
|
||||
}
|
||||
const query = queryBuilder.get();
|
||||
|
||||
const entries = await strapi.entityService.findMany(slugToProcess, query);
|
||||
|
||||
const data = convertData(entries, {
|
||||
slug: slugToProcess,
|
||||
dataFormat: exportFormat,
|
||||
relationsAsId,
|
||||
});
|
||||
|
||||
return data;
|
||||
};
|
||||
|
||||
const buildFilterQuery = (search) => {
|
||||
let { filters, sort: sortRaw } = qs.parse(search);
|
||||
|
||||
const [attr, value] = sortRaw?.split(':') || [];
|
||||
let sort = {};
|
||||
if (attr && value) {
|
||||
sort[attr] = value.toLowerCase();
|
||||
}
|
||||
|
||||
return {
|
||||
filters,
|
||||
sort,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Array<Object>} entries
|
||||
* @param {Object} options
|
||||
* @param {string} options.slug
|
||||
* @param {string} options.dataFormat
|
||||
* @param {boolean} options.relationsAsId
|
||||
* @returns
|
||||
*/
|
||||
const convertData = (entries, options) => {
|
||||
const converter = getConverter(options.dataFormat);
|
||||
|
||||
const convertedData = converter.convertEntries(entries, options);
|
||||
|
||||
return convertedData;
|
||||
};
|
||||
|
||||
const getConverter = (dataFormat) => {
|
||||
const converter = dataConverterConfigs[dataFormat];
|
||||
|
||||
if (!converter) {
|
||||
throw new Error(`Data format ${dataFormat} is not supported.`);
|
||||
}
|
||||
|
||||
return converter;
|
||||
};
|
||||
|
||||
const getPopulateFromSchema = (slug, deepness = 5) => {
|
||||
if (deepness <= 1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (slug === 'admin::user') {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const populate = {};
|
||||
const model = strapi.getModel(slug);
|
||||
for (const [attributeName, attribute] of Object.entries(getModelPopulationAttributes(model))) {
|
||||
if (!attribute) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attribute.type === 'component') {
|
||||
populate[attributeName] = getPopulateFromSchema(attribute.component, deepness - 1);
|
||||
} else if (attribute.type === 'dynamiczone') {
|
||||
const dynamicPopulate = attribute.components.reduce((zonePopulate, component) => {
|
||||
const compPopulate = getPopulateFromSchema(component, deepness - 1);
|
||||
return compPopulate === true ? zonePopulate : merge(zonePopulate, compPopulate);
|
||||
}, {});
|
||||
populate[attributeName] = isEmpty(dynamicPopulate) ? true : dynamicPopulate;
|
||||
} else if (attribute.type === 'relation') {
|
||||
const relationPopulate = getPopulateFromSchema(attribute.target, deepness - 1);
|
||||
if (relationPopulate) {
|
||||
populate[attributeName] = relationPopulate;
|
||||
}
|
||||
} else if (attribute.type === 'media') {
|
||||
populate[attributeName] = true;
|
||||
}
|
||||
}
|
||||
|
||||
return isEmpty(populate) ? true : { populate };
|
||||
};
|
||||
|
||||
const getModelPopulationAttributes = (model) => {
|
||||
if (model.uid === 'plugin::upload.file') {
|
||||
const { related, ...attributes } = model.attributes;
|
||||
return attributes;
|
||||
}
|
||||
|
||||
return model.attributes;
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
formats: dataFormats,
|
||||
exportData,
|
||||
getPopulateFromSchema,
|
||||
};
|
9
src/server/services/export/index.js
Normal file
9
src/server/services/export/index.js
Normal file
@ -0,0 +1,9 @@
|
||||
const { formats, exportData, getPopulateFromSchema } = require('./export');
|
||||
const { exportDataV2 } = require('./export-v2');
|
||||
|
||||
module.exports = {
|
||||
formats,
|
||||
exportData,
|
||||
getPopulateFromSchema,
|
||||
exportDataV2,
|
||||
};
|
327
src/server/services/import/import-v2.js
Normal file
327
src/server/services/import/import-v2.js
Normal file
@ -0,0 +1,327 @@
|
||||
const cloneDeep = require('lodash/cloneDeep');
|
||||
const isEmpty = require('lodash/isEmpty');
|
||||
const omit = require('lodash/omit');
|
||||
const pick = require('lodash/pick');
|
||||
const { isArraySafe } = require('../../../libs/arrays');
|
||||
|
||||
const { ObjectBuilder } = require('../../../libs/objects');
|
||||
const { CustomSlugs, CustomSlugToSlug } = require('../../config/constants');
|
||||
const { getModel, getModelAttributes, getModelConfig } = require('../../utils/models');
|
||||
const { findOrImportFile } = require('./utils/file');
|
||||
|
||||
/**
|
||||
* @typedef {Object} ImportDataRes
|
||||
* @property {Array<ImportDataFailures>} failures
|
||||
*/
|
||||
/**
|
||||
* Represents failed imports.
|
||||
* @typedef {Object} ImportDataFailures
|
||||
* @property {Error} error - Error raised.
|
||||
* @property {Object} data - Data for which import failed.
|
||||
*/
|
||||
/**
|
||||
* Import data.
|
||||
* @param {Object} fileContent - Content of the import file.
|
||||
* @param {Object} options
|
||||
* @param {string} options.slug - Slug of the imported model.
|
||||
* @param {Object} options.user - User importing the data.
|
||||
* @param {Object} options.idField - Field used as unique identifier.
|
||||
* @returns {Promise<ImportDataRes>}
|
||||
*/
|
||||
const importDataV2 = async (fileContent, { slug, user, idField }) => {
|
||||
const { data } = fileContent;
|
||||
|
||||
const slugs = Object.keys(data);
|
||||
let failures = [];
|
||||
// Import without setting relations.
|
||||
for (const slugFromFile of slugs) {
|
||||
let slugFailures = [];
|
||||
if (slugFromFile === CustomSlugToSlug[CustomSlugs.MEDIA]) {
|
||||
slugFailures = await importMedia(Object.values(data[slugFromFile]), { user }).then((res) => res.slugFailures);
|
||||
} else {
|
||||
slugFailures = await importOtherSlug(Object.values(data[slugFromFile]), {
|
||||
slug: slugFromFile,
|
||||
user,
|
||||
// Keep behavior of `idField` of version 1.
|
||||
...(slugFromFile === slug ? { idField } : {}),
|
||||
importStage: 'simpleAttributes',
|
||||
}).then((res) => res.failures);
|
||||
}
|
||||
failures = [...failures, ...(slugFailures || [])];
|
||||
}
|
||||
|
||||
// Set relations relations.
|
||||
for (const slugFromFile of slugs) {
|
||||
let slugFailures = [];
|
||||
if (slugFromFile === CustomSlugToSlug[CustomSlugs.MEDIA]) {
|
||||
slugFailures = await importMedia(Object.values(data[slugFromFile]), { user }).then((res) => res.slugFailures);
|
||||
} else {
|
||||
slugFailures = await importOtherSlug(Object.values(data[slugFromFile]), {
|
||||
slug: slugFromFile,
|
||||
user,
|
||||
// Keep behavior of `idField` of version 1.
|
||||
...(slugFromFile === slug ? { idField } : {}),
|
||||
importStage: 'relationAttributes',
|
||||
}).then((res) => res.failures);
|
||||
}
|
||||
failures = [...failures, ...(slugFailures || [])];
|
||||
}
|
||||
|
||||
// Sync primary key sequence for postgres databases.
|
||||
// See https://github.com/strapi/strapi/issues/12493.
|
||||
if (strapi.db.config.connection.client === 'postgres') {
|
||||
for (const slugFromFile of slugs) {
|
||||
const model = getModel(slugFromFile);
|
||||
await strapi.db.connection.raw(`SELECT SETVAL((SELECT PG_GET_SERIAL_SEQUENCE('${model.collectionName}', 'id')), (SELECT MAX(id) FROM ${model.collectionName}) + 1, FALSE);`);
|
||||
}
|
||||
}
|
||||
|
||||
return { failures };
|
||||
};
|
||||
|
||||
const importMedia = async (fileData, { user }) => {
|
||||
const processed = [];
|
||||
for (let fileDatum of fileData) {
|
||||
let res;
|
||||
try {
|
||||
await findOrImportFile(fileDatum, user, { allowedFileTypes: ['any'] });
|
||||
res = { success: true };
|
||||
} catch (err) {
|
||||
strapi.log.error(err);
|
||||
res = { success: false, error: err.message, args: [fileDatum] };
|
||||
}
|
||||
processed.push(res);
|
||||
}
|
||||
|
||||
const failures = processed.filter((p) => !p.success).map((f) => ({ error: f.error, data: f.args[0] }));
|
||||
|
||||
return {
|
||||
failures,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Import data.
|
||||
* @param {Array<Object>} data
|
||||
* @param {Object} importOptions
|
||||
* @param {('simpleAttributes'|'relationAttributes')} [importOptions.importStage]
|
||||
*/
|
||||
const importOtherSlug = async (data, { slug, user, idField, importStage }) => {
|
||||
// Sort localized data with default locale first.
|
||||
await (async () => {
|
||||
const { isLocalized } = getModelConfig(slug);
|
||||
|
||||
if (isLocalized) {
|
||||
const defaultLocale = await strapi.plugin('i18n').service('locales').getDefaultLocale();
|
||||
data = data.sort((dataA, dataB) => {
|
||||
if (dataA?.locale === defaultLocale && dataB?.locale === defaultLocale) {
|
||||
return 0;
|
||||
} else if (dataA?.locale === defaultLocale) {
|
||||
return -1;
|
||||
}
|
||||
return 1;
|
||||
});
|
||||
}
|
||||
})();
|
||||
|
||||
const processed = [];
|
||||
for (let datum of data) {
|
||||
let res;
|
||||
try {
|
||||
await updateOrCreate(user, slug, datum, idField, { importStage });
|
||||
res = { success: true };
|
||||
} catch (err) {
|
||||
strapi.log.error(err);
|
||||
res = { success: false, error: err.message, args: [datum] };
|
||||
}
|
||||
processed.push(res);
|
||||
}
|
||||
|
||||
const failures = processed.filter((p) => !p.success).map((f) => ({ error: f.error, data: f.args[0] }));
|
||||
|
||||
return {
|
||||
failures,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Update or create entries for a given model.
|
||||
* @param {Object} user - User importing the data.
|
||||
* @param {string} slug - Slug of the model.
|
||||
* @param {Object} datum - Data to update/create entries from.
|
||||
* @param {string} idField - Field used as unique identifier.
|
||||
* @param {Object} importOptions
|
||||
* @param {('simpleAttributes'|'relationAttributes')} [importOptions.importStage]
|
||||
*/
|
||||
const updateOrCreate = async (user, slug, datum, idField = 'id', { importStage }) => {
|
||||
datum = cloneDeep(datum);
|
||||
|
||||
if (importStage == 'simpleAttributes') {
|
||||
const attributeNames = getModelAttributes(slug, { filterOutType: ['component', 'dynamiczone', 'media', 'relation'], addIdAttribute: true })
|
||||
.map(({ name }) => name)
|
||||
.concat('localizations', 'locale');
|
||||
datum = pick(datum, attributeNames);
|
||||
} else if (importStage === 'relationAttributes') {
|
||||
const attributeNames = getModelAttributes(slug, { filterType: ['component', 'dynamiczone', 'media', 'relation'], addIdAttribute: true })
|
||||
.map(({ name }) => name)
|
||||
.concat('localizations', 'locale');
|
||||
datum = pick(datum, attributeNames);
|
||||
}
|
||||
|
||||
const model = getModel(slug);
|
||||
if (model.kind === 'singleType') {
|
||||
await updateOrCreateSingleType(user, slug, datum, { importStage });
|
||||
} else {
|
||||
await updateOrCreateCollectionType(user, slug, datum, { idField, importStage });
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Update or create entries for a given model.
|
||||
* @param {Object} user - User importing the data.
|
||||
* @param {string} slug - Slug of the model.
|
||||
* @param {Object} datum - Data to update/create entries from.
|
||||
* @param {Object} importOptions
|
||||
* @param {string} [importOptions.idField] - Field used as unique identifier.
|
||||
* @param {('simpleAttributes'|'relationAttributes')} [importOptions.importStage]
|
||||
*/
|
||||
const updateOrCreateCollectionType = async (user, slug, datum, { idField, importStage }) => {
|
||||
const { isLocalized } = getModelConfig(slug);
|
||||
|
||||
const whereBuilder = new ObjectBuilder();
|
||||
if (datum[idField]) {
|
||||
whereBuilder.extend({ [idField]: datum[idField] });
|
||||
}
|
||||
const where = whereBuilder.get();
|
||||
|
||||
if (!isLocalized) {
|
||||
let entry = await strapi.db.query(slug).findOne({ where });
|
||||
|
||||
if (!entry) {
|
||||
await strapi.entityService.create(slug, { data: datum });
|
||||
} else {
|
||||
await updateEntry(slug, entry.id, datum, { importStage });
|
||||
}
|
||||
} else {
|
||||
if (!datum.locale) {
|
||||
throw new Error(`No locale set to import entry for slug ${slug} (data ${JSON.stringify(datum)})`);
|
||||
}
|
||||
|
||||
const defaultLocale = await strapi.plugin('i18n').service('locales').getDefaultLocale();
|
||||
const isDatumInDefaultLocale = datum.locale === defaultLocale;
|
||||
|
||||
let entryDefaultLocale = null;
|
||||
let entry = await strapi.db.query(slug).findOne({ where, populate: ['localizations'] });
|
||||
if (isDatumInDefaultLocale) {
|
||||
entryDefaultLocale = entry;
|
||||
} else {
|
||||
if (entry) {
|
||||
// If `entry` has been found, `entry` holds the data for the default locale and
|
||||
// the data for other locales in its `localizations` attribute.
|
||||
const localizedEntries = [entry, ...(entry?.localizations || [])];
|
||||
entryDefaultLocale = localizedEntries.find((e) => e.locale === defaultLocale);
|
||||
entry = localizedEntries.find((e) => e.locale === datum.locale);
|
||||
} else {
|
||||
// Otherwise try to find entry for default locale through localized siblings.
|
||||
let localizationIdx = 0;
|
||||
const localizations = datum?.localizations || [];
|
||||
while (localizationIdx < localizations.length && !entryDefaultLocale && !entry) {
|
||||
const id = localizations[localizationIdx];
|
||||
const localizedEntry = await strapi.db.query(slug).findOne({ where: { id }, populate: ['localizations'] });
|
||||
const localizedEntries = localizedEntry != null ? [localizedEntry, ...(localizedEntry?.localizations || [])] : [];
|
||||
if (!entryDefaultLocale) {
|
||||
entryDefaultLocale = localizedEntries.find((e) => e.locale === defaultLocale);
|
||||
}
|
||||
if (!entry) {
|
||||
entry = localizedEntries.find((e) => e.locale === datum.locale);
|
||||
}
|
||||
localizationIdx += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
datum = omit(datum, ['localizations']);
|
||||
if (isEmpty(omit(datum, ['id']))) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (isDatumInDefaultLocale) {
|
||||
if (!entryDefaultLocale) {
|
||||
await strapi.entityService.create(slug, { data: datum });
|
||||
} else {
|
||||
await strapi.entityService.update(slug, entryDefaultLocale.id, { data: omit({ ...datum }, ['id']) });
|
||||
}
|
||||
} else {
|
||||
if (!entryDefaultLocale) {
|
||||
throw new Error(`Could not find default locale entry to import localization for slug ${slug} (data ${JSON.stringify(datum)})`);
|
||||
}
|
||||
|
||||
datum = omit({ ...datum }, ['id']);
|
||||
|
||||
if (!entry) {
|
||||
const createHandler = strapi.plugin('i18n').service('core-api').createCreateLocalizationHandler(getModel(slug));
|
||||
await createHandler({ id: entryDefaultLocale.id, data: datum });
|
||||
} else {
|
||||
await strapi.entityService.update(slug, entry.id, { data: datum });
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const updateOrCreateSingleType = async (user, slug, datum, { importStage }) => {
|
||||
const { isLocalized } = getModelConfig(slug);
|
||||
|
||||
if (!isLocalized) {
|
||||
let entry = await strapi.db.query(slug).findMany();
|
||||
entry = isArraySafe(entry) ? entry[0] : entry;
|
||||
|
||||
if (!entry) {
|
||||
await strapi.entityService.create(slug, { data: datum });
|
||||
} else {
|
||||
await updateEntry(slug, entry.id, datum, { importStage });
|
||||
}
|
||||
} else {
|
||||
const defaultLocale = await strapi.plugin('i18n').service('locales').getDefaultLocale();
|
||||
const isDatumInDefaultLocale = !datum.locale || datum.locale === defaultLocale;
|
||||
|
||||
datum = omit(datum, ['localizations']);
|
||||
if (isEmpty(omit(datum, ['id']))) {
|
||||
return;
|
||||
}
|
||||
|
||||
let entryDefaultLocale = await strapi.db.query(slug).findOne({ where: { locale: defaultLocale } });
|
||||
if (!entryDefaultLocale) {
|
||||
entryDefaultLocale = await strapi.entityService.create(slug, { data: { ...datum, locale: defaultLocale } });
|
||||
}
|
||||
|
||||
if (isDatumInDefaultLocale) {
|
||||
if (!entryDefaultLocale) {
|
||||
await strapi.entityService.create(slug, { data: datum });
|
||||
} else {
|
||||
await strapi.entityService.update(slug, entryDefaultLocale.id, { data: datum });
|
||||
}
|
||||
} else {
|
||||
const entryLocale = await strapi.db.query(slug).findOne({ where: { locale: datum.locale } });
|
||||
let datumLocale = { ...entryLocale, ...datum };
|
||||
|
||||
await strapi.db.query(slug).delete({ where: { locale: datum.locale } });
|
||||
|
||||
const createHandler = strapi.plugin('i18n').service('core-api').createCreateLocalizationHandler(getModel(slug));
|
||||
await createHandler({ id: entryDefaultLocale.id, data: datumLocale });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const updateEntry = async (slug, id, datum, { importStage }) => {
|
||||
datum = omit(datum, ['id']);
|
||||
|
||||
if (importStage === 'simpleAttributes') {
|
||||
await strapi.entityService.update(slug, id, { data: datum });
|
||||
} else if (importStage === 'relationAttributes') {
|
||||
await strapi.db.query(slug).update({ where: { id }, data: datum });
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
importDataV2,
|
||||
};
|
216
src/server/services/import/import.js
Normal file
216
src/server/services/import/import.js
Normal file
@ -0,0 +1,216 @@
|
||||
const { isArraySafe, toArray } = require('../../../libs/arrays');
|
||||
const { ObjectBuilder, isObjectSafe } = require('../../../libs/objects');
|
||||
const { CustomSlugs } = require('../../config/constants');
|
||||
const { getModelAttributes, getModel } = require('../../utils/models');
|
||||
const { findOrImportFile } = require('./utils/file');
|
||||
const { parseInputData } = require('./parsers');
|
||||
|
||||
/**
|
||||
* @typedef {Object} ImportDataRes
|
||||
* @property {Array<ImportDataFailures>} failures
|
||||
*/
|
||||
/**
|
||||
* Represents failed imports.
|
||||
* @typedef {Object} ImportDataFailures
|
||||
* @property {Error} error - Error raised.
|
||||
* @property {Object} data - Data for which import failed.
|
||||
*/
|
||||
/**
|
||||
* Import data.
|
||||
* @param {Array<Object>} dataRaw - Data to import.
|
||||
* @param {Object} options
|
||||
* @param {string} options.slug - Slug of the model to import.
|
||||
* @param {("csv" | "json")} options.format - Format of the imported data.
|
||||
* @param {Object} options.user - User importing the data.
|
||||
* @param {Object} options.idField - Field used as unique identifier.
|
||||
* @returns {Promise<ImportDataRes>}
|
||||
*/
|
||||
const importData = async (dataRaw, { slug, format, user, idField }) => {
|
||||
let data = await parseInputData(format, dataRaw, { slug });
|
||||
data = toArray(data);
|
||||
|
||||
let res;
|
||||
if (slug === CustomSlugs.MEDIA) {
|
||||
res = await importMedia(data, { user });
|
||||
} else {
|
||||
res = await importOtherSlug(data, { slug, user, idField });
|
||||
}
|
||||
|
||||
return res;
|
||||
};
|
||||
|
||||
const importMedia = async (fileData, { user }) => {
|
||||
const processed = [];
|
||||
for (let fileDatum of fileData) {
|
||||
let res;
|
||||
try {
|
||||
await findOrImportFile(fileDatum, user, { allowedFileTypes: ['any'] });
|
||||
res = { success: true };
|
||||
} catch (err) {
|
||||
strapi.log.error(err);
|
||||
res = { success: false, error: err.message, args: [fileDatum] };
|
||||
}
|
||||
processed.push(res);
|
||||
}
|
||||
|
||||
const failures = processed.filter((p) => !p.success).map((f) => ({ error: f.error, data: f.args[0] }));
|
||||
|
||||
return {
|
||||
failures,
|
||||
};
|
||||
};
|
||||
|
||||
const importOtherSlug = async (data, { slug, user, idField }) => {
|
||||
const processed = [];
|
||||
for (let datum of data) {
|
||||
let res;
|
||||
try {
|
||||
await updateOrCreate(user, slug, datum, idField);
|
||||
res = { success: true };
|
||||
} catch (err) {
|
||||
strapi.log.error(err);
|
||||
res = { success: false, error: err.message, args: [datum] };
|
||||
}
|
||||
processed.push(res);
|
||||
}
|
||||
|
||||
const failures = processed.filter((p) => !p.success).map((f) => ({ error: f.error, data: f.args[0] }));
|
||||
|
||||
return {
|
||||
failures,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Update or create entries for a given model.
|
||||
* @param {Object} user - User importing the data.
|
||||
* @param {string} slug - Slug of the model.
|
||||
* @param {Object} data - Data to update/create entries from.
|
||||
* @param {string} idField - Field used as unique identifier.
|
||||
* @returns Updated/created entry.
|
||||
*/
|
||||
const updateOrCreate = async (user, slug, data, idField = 'id') => {
|
||||
const relationAttributes = getModelAttributes(slug, { filterType: ['component', 'dynamiczone', 'media', 'relation'] });
|
||||
for (let attribute of relationAttributes) {
|
||||
data[attribute.name] = await updateOrCreateRelation(user, attribute, data[attribute.name]);
|
||||
}
|
||||
|
||||
let entry;
|
||||
const model = getModel(slug);
|
||||
if (model.kind === 'singleType') {
|
||||
entry = await updateOrCreateSingleType(user, slug, data, idField);
|
||||
} else {
|
||||
entry = await updateOrCreateCollectionType(user, slug, data, idField);
|
||||
}
|
||||
return entry;
|
||||
};
|
||||
|
||||
const updateOrCreateCollectionType = async (user, slug, data, idField) => {
|
||||
const whereBuilder = new ObjectBuilder();
|
||||
if (data[idField]) {
|
||||
whereBuilder.extend({ [idField]: data[idField] });
|
||||
}
|
||||
const where = whereBuilder.get();
|
||||
|
||||
// Prevent strapi from throwing a unique constraint error on id field.
|
||||
if (idField !== 'id') {
|
||||
delete data.id;
|
||||
}
|
||||
|
||||
let entry;
|
||||
if (!where[idField]) {
|
||||
entry = await strapi.db.query(slug).create({ data });
|
||||
} else {
|
||||
entry = await strapi.db.query(slug).update({ where, data });
|
||||
|
||||
if (!entry) {
|
||||
entry = await strapi.db.query(slug).create({ data });
|
||||
}
|
||||
}
|
||||
|
||||
return entry;
|
||||
};
|
||||
|
||||
const updateOrCreateSingleType = async (user, slug, data, idField) => {
|
||||
delete data.id;
|
||||
|
||||
let [entry] = await strapi.db.query(slug).findMany();
|
||||
if (!entry) {
|
||||
entry = await strapi.db.query(slug).create({ data });
|
||||
} else {
|
||||
entry = await strapi.db.query(slug).update({ where: { id: entry.id }, data });
|
||||
}
|
||||
|
||||
return entry;
|
||||
};
|
||||
|
||||
/**
|
||||
* Update or create a relation.
|
||||
* @param {Object} user
|
||||
* @param {Attribute} rel
|
||||
* @param {number | Object | Array<Object>} relData
|
||||
*/
|
||||
const updateOrCreateRelation = async (user, rel, relData) => {
|
||||
if (relData == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (['createdBy', 'updatedBy'].includes(rel.name)) {
|
||||
return user.id;
|
||||
} else if (rel.type === 'dynamiczone') {
|
||||
const components = [];
|
||||
for (const componentDatum of relData || []) {
|
||||
let component = await updateOrCreate(user, componentDatum.__component, componentDatum);
|
||||
component = { ...component, __component: componentDatum.__component };
|
||||
components.push(component);
|
||||
}
|
||||
return components;
|
||||
} else if (rel.type === 'component') {
|
||||
relData = toArray(relData);
|
||||
relData = rel.repeatable ? relData : relData.slice(0, 1);
|
||||
const entryIds = [];
|
||||
for (const relDatum of relData) {
|
||||
if (typeof relDatum === 'number') {
|
||||
entryIds.push(relDatum);
|
||||
} else if (isObjectSafe(relDatum)) {
|
||||
const entry = await updateOrCreate(user, rel.component, relDatum);
|
||||
if (entry?.id) {
|
||||
entryIds.push(entry.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
return rel.repeatable ? entryIds : entryIds?.[0] || null;
|
||||
} else if (rel.type === 'media') {
|
||||
relData = toArray(relData);
|
||||
relData = rel.multiple ? relData : relData.slice(0, 1);
|
||||
const entryIds = [];
|
||||
for (const relDatum of relData) {
|
||||
const media = await findOrImportFile(relDatum, user, { allowedFileTypes: rel.allowedTypes });
|
||||
if (media?.id) {
|
||||
entryIds.push(media.id);
|
||||
}
|
||||
}
|
||||
return rel.multiple ? entryIds : entryIds?.[0] || null;
|
||||
} else if (rel.type === 'relation') {
|
||||
const isMultiple = isArraySafe(relData);
|
||||
relData = toArray(relData);
|
||||
const entryIds = [];
|
||||
for (const relDatum of relData) {
|
||||
if (typeof relDatum === 'number') {
|
||||
entryIds.push(relDatum);
|
||||
} else if (isObjectSafe(relDatum)) {
|
||||
const entry = await updateOrCreate(user, rel.target, relDatum);
|
||||
if (entry?.id) {
|
||||
entryIds.push(entry.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
return isMultiple ? entryIds : entryIds?.[0] || null;
|
||||
}
|
||||
|
||||
throw new Error(`Could not update or create relation of type ${rel.type}.`);
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
importData,
|
||||
};
|
9
src/server/services/import/index.js
Normal file
9
src/server/services/import/index.js
Normal file
@ -0,0 +1,9 @@
|
||||
const { importData } = require('./import');
|
||||
const { importDataV2 } = require('./import-v2');
|
||||
const { parseInputData } = require('./parsers');
|
||||
|
||||
module.exports = {
|
||||
importData,
|
||||
importDataV2,
|
||||
parseInputData,
|
||||
};
|
65
src/server/services/import/parsers.js
Normal file
65
src/server/services/import/parsers.js
Normal file
@ -0,0 +1,65 @@
|
||||
'use strict';
|
||||
|
||||
const csvtojson = require('csvtojson');
|
||||
|
||||
const { isArraySafe } = require('../../../libs/arrays');
|
||||
const { isObjectSafe } = require('../../../libs/objects');
|
||||
const { getModelAttributes } = require('../../utils/models');
|
||||
|
||||
const parseCsv = async (dataRaw, { slug }) => {
|
||||
let data = await csvtojson().fromString(dataRaw);
|
||||
|
||||
const relationNames = getModelAttributes(slug, { filterType: ['component', 'dynamiczone', 'media', 'relation'] }).map((a) => a.name);
|
||||
data = data.map((datum) => {
|
||||
for (let name of relationNames) {
|
||||
try {
|
||||
datum[name] = JSON.parse(datum[name]);
|
||||
} catch (err) {
|
||||
strapi.log.error(err);
|
||||
}
|
||||
}
|
||||
return datum;
|
||||
});
|
||||
|
||||
return data;
|
||||
};
|
||||
|
||||
const parseJson = async (dataRaw) => {
|
||||
let data = JSON.parse(dataRaw);
|
||||
return data;
|
||||
};
|
||||
|
||||
const parseJso = async (dataRaw) => {
|
||||
if (!isObjectSafe(dataRaw) && !isArraySafe(dataRaw)) {
|
||||
throw new Error(`To import JSO, data must be an array or an object`);
|
||||
}
|
||||
|
||||
return dataRaw;
|
||||
};
|
||||
|
||||
const parsers = {
|
||||
csv: parseCsv,
|
||||
jso: parseJso,
|
||||
json: parseJson,
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse input data.
|
||||
* @param {("csv"|"jso"|"json")} format
|
||||
* @param {*} dataRaw
|
||||
* @param {Object} options
|
||||
* @param {string} options.slug
|
||||
*/
|
||||
const parseInputData = async (format, dataRaw, { slug }) => {
|
||||
const parser = parsers[format];
|
||||
if (!parser) {
|
||||
throw new Error(`Data input format ${format} is not supported.`);
|
||||
}
|
||||
|
||||
const data = await parser(dataRaw, { slug });
|
||||
return data;
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
parseInputData,
|
||||
};
|
226
src/server/services/import/utils/file.js
Normal file
226
src/server/services/import/utils/file.js
Normal file
@ -0,0 +1,226 @@
|
||||
const fs = require('fs');
|
||||
const fse = require('fs-extra');
|
||||
const last = require('lodash/last');
|
||||
const trim = require('lodash/trim');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
const { isObjectSafe } = require('../../../../libs/objects');
|
||||
|
||||
/**
|
||||
* Find or import a file.
|
||||
* @param {*} fileData - Strapi file data.
|
||||
* @param {*} user - Strapi user.
|
||||
* @param {Object} options
|
||||
* @param {Array<string>} options.allowedFileTypes - File types the file should match (see Strapi file allowedTypes).
|
||||
* @returns
|
||||
*/
|
||||
const findOrImportFile = async (fileData, user, { allowedFileTypes }) => {
|
||||
let obj = {};
|
||||
if (typeof fileData === 'number') {
|
||||
obj.id = fileData;
|
||||
} else if (typeof fileData === 'string') {
|
||||
obj.url = fileData;
|
||||
} else if (isObjectSafe(fileData)) {
|
||||
obj = fileData;
|
||||
} else {
|
||||
throw new Error(`Invalid data format '${typeof fileData}' to import media. Only 'string', 'number', 'object' are accepted.`);
|
||||
}
|
||||
|
||||
let file = await findFile(obj, user, allowedFileTypes);
|
||||
|
||||
if (file && !isExtensionAllowed(file.ext.substring(1), allowedFileTypes)) {
|
||||
file = null;
|
||||
}
|
||||
|
||||
return file;
|
||||
};
|
||||
|
||||
/**
|
||||
* Find a file.
|
||||
* @param {Object} filters
|
||||
* @param {number} [filters.id] - File id.
|
||||
* @param {string} [filters.hash] - File hash.
|
||||
* @param {string} [filters.name] - File name.
|
||||
* @param {string} [filters.url]
|
||||
* @param {string} [filters.alternativeText]
|
||||
* @param {string} [filters.caption]
|
||||
* @param {Object} user
|
||||
* @returns
|
||||
*/
|
||||
const findFile = async ({ id, hash, name, url, alternativeText, caption }, user, allowedFileTypes) => {
|
||||
let file = null;
|
||||
|
||||
if (!file && id) {
|
||||
file = await strapi.entityService.findOne('plugin::upload.file', id);
|
||||
}
|
||||
if (!file && hash) {
|
||||
[file] = await strapi.entityService.findMany('plugin::upload.file', { filters: { hash }, limit: 1 });
|
||||
}
|
||||
if (!file && name) {
|
||||
[file] = await strapi.entityService.findMany('plugin::upload.file', { filters: { name }, limit: 1 });
|
||||
}
|
||||
if (!file && url) {
|
||||
const checkResult = isValidFileUrl(url, allowedFileTypes);
|
||||
if (checkResult.isValid) {
|
||||
file = await findFile({ hash: checkResult.fileData.hash, name: checkResult.fileData.fileName }, user, allowedFileTypes);
|
||||
|
||||
if (!file) {
|
||||
file = await importFile({ id, url: checkResult.fileData.rawUrl, name, alternativeText, caption }, user);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return file;
|
||||
};
|
||||
|
||||
const importFile = async ({ id, url, name, alternativeText, caption }, user) => {
|
||||
let file;
|
||||
try {
|
||||
file = await fetchFile(url);
|
||||
|
||||
let [uploadedFile] = await strapi
|
||||
.plugin('upload')
|
||||
.service('upload')
|
||||
.upload(
|
||||
{
|
||||
files: {
|
||||
name: file.name,
|
||||
type: file.type,
|
||||
size: file.size,
|
||||
path: file.path,
|
||||
},
|
||||
data: {
|
||||
fileInfo: {
|
||||
name: name || file.name,
|
||||
alternativeText: alternativeText || '',
|
||||
caption: caption || '',
|
||||
},
|
||||
},
|
||||
},
|
||||
{ user },
|
||||
);
|
||||
|
||||
if (id) {
|
||||
uploadedFile = await strapi.db.query('plugin::upload.file').update({
|
||||
where: { id: uploadedFile.id },
|
||||
data: { id },
|
||||
});
|
||||
}
|
||||
|
||||
return uploadedFile;
|
||||
} catch (err) {
|
||||
strapi.log.error(err);
|
||||
throw err;
|
||||
} finally {
|
||||
deleteFileIfExists(file?.path);
|
||||
}
|
||||
};
|
||||
|
||||
const fetchFile = async (url) => {
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
const contentType = response.headers.get('content-type').split(';')[0];
|
||||
const contentLength = parseInt(response.headers.get('content-length')) || 0;
|
||||
const buffer = await response.buffer();
|
||||
const fileData = getFileDataFromRawUrl(url);
|
||||
const filePath = await writeFile(fileData.name, buffer);
|
||||
return {
|
||||
name: fileData.name,
|
||||
type: contentType,
|
||||
size: contentLength,
|
||||
path: filePath,
|
||||
};
|
||||
} catch (error) {
|
||||
throw new Error(`Tried to fetch file from url ${url} but failed with error: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
const writeFile = async (name, content) => {
|
||||
const tmpWorkingDirectory = await fse.mkdtemp(path.join(os.tmpdir(), 'strapi-upload-'));
|
||||
|
||||
const filePath = path.join(tmpWorkingDirectory, name);
|
||||
try {
|
||||
fs.writeFileSync(filePath, content);
|
||||
return filePath;
|
||||
} catch (err) {
|
||||
strapi.log.error(err);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
const deleteFileIfExists = (filePath) => {
|
||||
if (filePath && fs.existsSync(filePath)) {
|
||||
fs.rmSync(filePath);
|
||||
}
|
||||
};
|
||||
|
||||
const isValidFileUrl = (url, allowedFileTypes) => {
|
||||
try {
|
||||
const fileData = getFileDataFromRawUrl(url);
|
||||
|
||||
return {
|
||||
isValid: isExtensionAllowed(fileData.extension, allowedFileTypes),
|
||||
fileData: {
|
||||
hash: fileData.hash,
|
||||
fileName: fileData.name,
|
||||
rawUrl: url,
|
||||
},
|
||||
};
|
||||
} catch (err) {
|
||||
strapi.log.error(err);
|
||||
return {
|
||||
isValid: false,
|
||||
fileData: {
|
||||
hash: '',
|
||||
fileName: '',
|
||||
rawUrl: '',
|
||||
},
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const isExtensionAllowed = (ext, allowedFileTypes) => {
|
||||
const checkers = allowedFileTypes.map(getFileTypeChecker);
|
||||
return checkers.some((checker) => checker(ext));
|
||||
};
|
||||
|
||||
const ALLOWED_AUDIOS = ['mp3', 'wav', 'ogg'];
|
||||
const ALLOWED_IMAGES = ['png', 'gif', 'jpg', 'jpeg', 'svg', 'bmp', 'tif', 'tiff'];
|
||||
const ALLOWED_VIDEOS = ['mp4', 'avi'];
|
||||
|
||||
/** See Strapi file allowedTypes for object keys. */
|
||||
const fileTypeCheckers = {
|
||||
any: (ext) => true,
|
||||
audios: (ext) => ALLOWED_AUDIOS.includes(ext),
|
||||
files: (ext) => true,
|
||||
images: (ext) => ALLOWED_IMAGES.includes(ext),
|
||||
videos: (ext) => ALLOWED_VIDEOS.includes(ext),
|
||||
};
|
||||
|
||||
const getFileTypeChecker = (type) => {
|
||||
const checker = fileTypeCheckers[type];
|
||||
if (!checker) {
|
||||
throw new Error(`Strapi file type ${type} not handled.`);
|
||||
}
|
||||
return checker;
|
||||
};
|
||||
|
||||
const getFileDataFromRawUrl = (rawUrl) => {
|
||||
const parsedUrl = new URL(decodeURIComponent(rawUrl));
|
||||
|
||||
const name = trim(parsedUrl.pathname, '/').replace(/\//g, '-');
|
||||
const extension = parsedUrl.pathname.split('.').pop().toLowerCase();
|
||||
const hash = last(parsedUrl.pathname.split('/')).slice(0, -(extension.length + 1));
|
||||
|
||||
return {
|
||||
hash,
|
||||
name,
|
||||
extension,
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
findOrImportFile,
|
||||
};
|
9
src/server/services/index.js
Normal file
9
src/server/services/index.js
Normal file
@ -0,0 +1,9 @@
|
||||
'use strict';
|
||||
|
||||
const exportService = require('./export');
|
||||
const importService = require('./import');
|
||||
|
||||
module.exports = {
|
||||
export: exportService,
|
||||
import: importService,
|
||||
};
|
28
src/server/utils/errors.js
Normal file
28
src/server/utils/errors.js
Normal file
@ -0,0 +1,28 @@
|
||||
const _ = require('lodash');
|
||||
|
||||
const errorCodes = {};
|
||||
|
||||
const errorMessages = {};
|
||||
|
||||
Object.keys(errorMessages).forEach(
|
||||
(k) =>
|
||||
(errorMessages[k] = _.template(errorMessages[k], {
|
||||
interpolate: /\{\s*(\S+)\s*\}/g,
|
||||
})),
|
||||
);
|
||||
|
||||
class BusinessError extends Error {
|
||||
constructor(errorCodeOrMessage, interpolations) {
|
||||
const isErrorCode = !!errorCodes[errorCodeOrMessage];
|
||||
|
||||
super(isErrorCode ? errorMessages[errorCodeOrMessage](interpolations) : errorCodeOrMessage);
|
||||
|
||||
this.name = this.constructor.name;
|
||||
this.code = isErrorCode ? errorCodeOrMessage : 'UNDEFINED';
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
BusinessError,
|
||||
errorCodes,
|
||||
};
|
17
src/server/utils/getConfig.js
Normal file
17
src/server/utils/getConfig.js
Normal file
@ -0,0 +1,17 @@
|
||||
const pluginId = require('./pluginId');
|
||||
|
||||
/**
|
||||
* @typedef {("serverPublicHostname")} ConfigParam
|
||||
*/
|
||||
|
||||
/**
|
||||
* Get a config parameter.
|
||||
* @param {ConfigParam} param
|
||||
*/
|
||||
const getConfig = (param) => {
|
||||
return strapi.config.get(`plugin.${pluginId}.${param}`);
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
getConfig,
|
||||
};
|
19
src/server/utils/index.js
Normal file
19
src/server/utils/index.js
Normal file
@ -0,0 +1,19 @@
|
||||
const pluginId = require('./pluginId');
|
||||
|
||||
/**
|
||||
* ServiceName.
|
||||
* @typedef {("export"|"import")} ServiceName
|
||||
*/
|
||||
|
||||
/**
|
||||
* Get a plugin service.
|
||||
* @param {ServiceName} serviceName
|
||||
* @returns
|
||||
*/
|
||||
const getService = (serviceName) => {
|
||||
return strapi.plugin(pluginId).service(serviceName);
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
getService,
|
||||
};
|
121
src/server/utils/models.js
Normal file
121
src/server/utils/models.js
Normal file
@ -0,0 +1,121 @@
|
||||
'use strict';
|
||||
|
||||
const { toArray } = require('../../libs/arrays');
|
||||
|
||||
/**
|
||||
* ModelKind.
|
||||
* @typedef {("collectionType"|"singleType")} ModelKind
|
||||
*/
|
||||
|
||||
/**
|
||||
* A model.
|
||||
* @typedef {Object} Model
|
||||
* @property {ModelKind} kind
|
||||
* @property {string} collectionName
|
||||
* @property {{[k: string]: Attribute}} attributes - Name of the attribute.
|
||||
*/
|
||||
|
||||
/**
|
||||
* AttributeType.
|
||||
* @typedef {("boolean"|"component"|"datetime"|"dynamiczone"|"increments"|"media"|"number"|"relation"|"string"|"text")} AttributeType
|
||||
*/
|
||||
|
||||
/**
|
||||
* AttributeTarget.
|
||||
* @typedef {("admin::user")} AttributeTarget
|
||||
*/
|
||||
|
||||
/**
|
||||
* An attribute of a model.
|
||||
* @typedef {Object} Attribute
|
||||
* @property {AttributeType} type
|
||||
* @property {string} name - Name of the attribute.
|
||||
* @property {string} [target] - Slug of the target model (if type is 'relation').
|
||||
* @property {string} [component] - Name of the targetted component.
|
||||
* @property {boolean} [repeatable] - Whether the component is repeatable.
|
||||
* @property {Array<string>} [components] - Component names of the dynamic zone.
|
||||
* @property {("audios"|"files"|"images"|"videos")} [allowedTypes] - Allowed file types.
|
||||
* @property {boolean} [multiple] - Whether there are multiple files.
|
||||
*/
|
||||
|
||||
const getAllSlugs = () => {
|
||||
return Array.from(strapi.db.metadata)
|
||||
.map(([collectionName]) => collectionName)
|
||||
.filter((collectionName) => collectionName.startsWith('api::'));
|
||||
};
|
||||
|
||||
/**
|
||||
* Get a model.
|
||||
* @param {string} slug - Slug of the model.
|
||||
* @returns {Model}
|
||||
*/
|
||||
const getModel = (slug) => {
|
||||
return strapi.getModel(slug);
|
||||
};
|
||||
|
||||
const getModelConfig = (modelOrSlug) => {
|
||||
const model = getModelFromSlugOrModel(modelOrSlug);
|
||||
|
||||
return {
|
||||
kind: model.kind === 'singleType' ? 'single' : 'collection',
|
||||
isLocalized: !!model.pluginOptions?.i18n?.localized,
|
||||
};
|
||||
};
|
||||
|
||||
const getModelFromSlugOrModel = (modelOrSlug) => {
|
||||
let model = modelOrSlug;
|
||||
if (typeof model === 'string') {
|
||||
model = getModel(modelOrSlug);
|
||||
}
|
||||
|
||||
return model;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get the attributes of a model.
|
||||
* @param {string} slug - Slug of the model.
|
||||
* @param {Object} options
|
||||
* @param {AttributeType | Array<AttributeType>} [options.filterType] - Only attributes matching the type(s) will be kept.
|
||||
* @param {AttributeType | Array<AttributeType>} [options.filterOutType] - Remove attributes matching the specified type(s).
|
||||
* @param {AttributeTarget | Array<AttributeTarget>} [options.filterOutTarget] - Remove attributes matching the specified target(s).
|
||||
* @param {boolean} [options.addIdAttribute] - Add `id` in the returned attributes.
|
||||
* @returns {Array<Attribute>}
|
||||
*/
|
||||
const getModelAttributes = (slug, options = {}) => {
|
||||
const typesToKeep = options.filterType ? toArray(options.filterType) : [];
|
||||
const typesToFilterOut = options.filterOutType ? toArray(options.filterOutType) : [];
|
||||
const filterOutTarget = toArray(options.filterOutTarget || []);
|
||||
|
||||
const attributesObj = strapi.getModel(slug).attributes;
|
||||
let attributes = Object.keys(attributesObj)
|
||||
.reduce((acc, key) => acc.concat({ ...attributesObj[key], name: key }), [])
|
||||
.filter((attr) => !typesToFilterOut.includes(attr.type))
|
||||
.filter((attr) => !filterOutTarget.includes(attr.target));
|
||||
|
||||
if (typesToKeep.length) {
|
||||
attributes = attributes.filter((attr) => typesToKeep.includes(attr.type));
|
||||
}
|
||||
|
||||
if (options.addIdAttribute) {
|
||||
attributes.unshift({ name: 'id' });
|
||||
}
|
||||
|
||||
return attributes;
|
||||
};
|
||||
|
||||
/**
|
||||
* Indicate whether an attribute is a dynamic zone.
|
||||
* @param {Attribute} attribute
|
||||
* @returns {boolean}
|
||||
*/
|
||||
const isAttributeDynamicZone = (attribute) => {
|
||||
return attribute.components && Array.isArray(attribute.components);
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
getAllSlugs,
|
||||
getModel,
|
||||
getModelConfig,
|
||||
getModelAttributes,
|
||||
isAttributeDynamicZone,
|
||||
};
|
5
src/server/utils/pluginId.js
Normal file
5
src/server/utils/pluginId.js
Normal file
@ -0,0 +1,5 @@
|
||||
const pluginPkg = require('../../package.json');
|
||||
|
||||
const pluginId = pluginPkg.name.replace(/^(@[^-,.][\w,-]+\/|strapi-)plugin-/i, '');
|
||||
|
||||
module.exports = pluginId;
|
16
tsconfig.json
Normal file
16
tsconfig.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES6",
|
||||
"module": "CommonJS",
|
||||
"strict": true,
|
||||
"allowJs": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"rootDir": "./src",
|
||||
"outDir": "./"
|
||||
},
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
@ -12003,6 +12003,11 @@ typescript@4.6.2:
|
||||
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4"
|
||||
integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg==
|
||||
|
||||
typescript@^5.0.3:
|
||||
version "5.0.3"
|
||||
resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.0.3.tgz#fe976f0c826a88d0a382007681cbb2da44afdedf"
|
||||
integrity sha512-xv8mOEDnigb/tN9PSMTwSEqAnUvkoXMQlicOb0IUVDBSQCgBSaAAROUZYy2IcUy5qU6XajK5jjjO7TMWqBTKZA==
|
||||
|
||||
uc.micro@^1.0.1, uc.micro@^1.0.5:
|
||||
version "1.0.6"
|
||||
resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
|
||||
|
Loading…
x
Reference in New Issue
Block a user