chore: Setup typescript for server

This commit is contained in:
Baptiste Studer 2023-04-02 17:44:34 +02:00
parent fb89c4efcb
commit 375a341aac
48 changed files with 2330 additions and 3 deletions

View File

@ -17,8 +17,7 @@
"server/"
],
"scripts": {
"lint:check": "eslint . && yarn prettier --check .",
"lint:fix": "eslint . --fix && yarn prettier --write .",
"build": "tsc --build",
"test": "jest --forceExit --detectOpenHandles --runInBand",
"test:watch": "jest --forceExit --detectOpenHandles --runInBand --watch",
"release": "standard-version",
@ -54,7 +53,8 @@
"prettier": "^2.7.1",
"sqlite3": "^5.0.11",
"standard-version": "^9.5.0",
"supertest": "^6.2.4"
"supertest": "^6.2.4",
"typescript": "^5.0.3"
},
"peerDependencies": {
"@strapi/strapi": "^4.0.0"

20
src/libs/arrays.ts Normal file
View File

@ -0,0 +1,20 @@
/**
* Check if value is an array.
*/
const isArraySafe = <T>(val: T | T[]): val is T[] => {
return val && Array.isArray(val);
};
/**
* Convert value to array if not already.
* @param {*} val
* @returns {Array<*>}
*/
const toArray = <T>(val: T | T[]): T[] => {
return isArraySafe(val) ? val : [val];
};
module.exports = {
isArraySafe,
toArray,
};

52
src/libs/objects.js Normal file
View File

@ -0,0 +1,52 @@
const deepmerge = require('deepmerge');
class ObjectBuilder {
_obj = {};
get() {
return this._obj;
}
extend(obj) {
this._obj = { ...this._obj, ...obj };
}
}
/**
* Check if value is an object.
* @param {*} val
* @returns {boolean}
*/
const isObjectSafe = (val) => {
return val && !Array.isArray(val) && typeof val === 'object';
};
const isObjectEmpty = (obj) => {
for (let i in obj) {
return false;
}
return true;
};
const mergeObjects = (x, y) => {
return deepmerge(x, y, {
arrayMerge: (target, source) => {
source.forEach((item) => {
if (target.indexOf(item) === -1) {
target.push(item);
}
});
return target;
},
});
};
const logObj = (obj) => JSON.stringify(obj, null, ' ');
module.exports = {
ObjectBuilder,
logObj,
isObjectSafe,
isObjectEmpty,
mergeObjects,
};

5
src/server/bootstrap.js vendored Normal file
View File

@ -0,0 +1,5 @@
'use strict';
module.exports = ({ strapi }) => {
// bootstrap phase
};

View File

@ -0,0 +1,18 @@
const CustomSlugs = {
MEDIA: 'media',
WHOLE_DB: 'custom:db',
};
const CustomSlugToSlug = {
[CustomSlugs.MEDIA]: 'plugin::upload.file',
};
const isCustomSlug = (slug) => {
return !!CustomSlugToSlug[slug];
};
module.exports = {
CustomSlugs,
CustomSlugToSlug,
isCustomSlug,
};

View File

@ -0,0 +1,15 @@
'use strict';
module.exports = {
default: {
/**
* Public hostname of the server.
*/
serverPublicHostname: '',
},
validator: ({ serverPublicHostname } = {}) => {
if (typeof serverPublicHostname !== 'string') {
throw new Error('serverPublicHostname has to be a string.');
}
},
};

View File

@ -0,0 +1,3 @@
'use strict';
module.exports = {};

View File

@ -0,0 +1,43 @@
'use strict';
const { CustomSlugs } = require('../../../config/constants');
const { getService } = require('../../../utils');
const { getAllSlugs } = require('../../../utils/models');
const { handleAsyncError } = require('../../content-api/utils');
const exportData = async (ctx) => {
if (!hasPermissions(ctx)) {
return ctx.forbidden();
}
let { slug, search, applySearch, exportFormat, relationsAsId, deepness = 5 } = ctx.request.body;
let data;
if (exportFormat === getService('export').formats.JSON_V2) {
data = await getService('export').exportDataV2({ slug, search, applySearch, deepness });
} else {
data = await getService('export').exportData({ slug, search, applySearch, exportFormat, relationsAsId, deepness });
}
ctx.body = {
data,
};
};
const hasPermissions = (ctx) => {
let { slug } = ctx.request.body;
const { userAbility } = ctx.state;
const slugs = slug === CustomSlugs.WHOLE_DB ? getAllSlugs() : [slug];
const allowedSlugs = slugs.filter((slug) => {
const permissionChecker = strapi.plugin('content-manager').service('permission-checker').create({ userAbility, model: slug });
return permissionChecker.can.read();
});
return !!allowedSlugs.length;
};
module.exports = ({ strapi }) => ({
exportData: handleAsyncError(exportData),
});

View File

@ -0,0 +1 @@
module.exports = require('./export-controller');

View File

@ -0,0 +1,28 @@
'use strict';
const { getModelAttributes } = require('../../../utils/models');
const getModelAttributesEndpoint = async (ctx) => {
const { slug } = ctx.params;
const attributeNames = getModelAttributes(slug)
.filter(filterAttribute)
.map((attr) => attr.name);
ctx.body = {
data: {
attribute_names: attributeNames,
},
};
};
const filterAttribute = (attr) => {
const filters = [filterType, filterName];
return filters.every((filter) => filter(attr));
};
const filterType = (attr) => !['relation', 'component', 'dynamiczone'].includes(attr.type);
const filterName = (attr) => !['createdAt', 'updatedAt', 'publishedAt', 'locale'].includes(attr.name);
module.exports = ({ strapi }) => getModelAttributesEndpoint;

View File

@ -0,0 +1,9 @@
'use strict';
const getModelAttributes = require('./get-model-attributes');
const importData = require('./import-data');
module.exports = ({ strapi }) => ({
getModelAttributes: getModelAttributes({ strapi }),
importData: importData({ strapi }),
});

View File

@ -0,0 +1,45 @@
'use strict';
const { getService } = require('../../../utils');
const importData = async (ctx) => {
if (!hasPermissions(ctx)) {
return ctx.forbidden();
}
const { user } = ctx.state;
const { slug, data: dataRaw, format, idField } = ctx.request.body;
const fileContent = await getService('import').parseInputData(format, dataRaw, { slug });
let res;
if (fileContent?.version === 2) {
res = await getService('import').importDataV2(fileContent, {
slug,
user,
idField,
});
} else {
res = await getService('import').importData(dataRaw, {
slug,
format,
user,
idField,
});
}
ctx.body = {
failures: res.failures,
};
};
const hasPermissions = (ctx) => {
let { slug } = ctx.request.body;
const { userAbility } = ctx.state;
const permissionChecker = strapi.plugin('content-manager').service('permission-checker').create({ userAbility, model: slug });
return permissionChecker.can.create() && permissionChecker.can.update();
};
module.exports = ({ strapi }) => importData;

View File

@ -0,0 +1 @@
module.exports = require('./import-controller');

View File

@ -0,0 +1,34 @@
'use strict';
const Joi = require('joi');
const { getService } = require('../../../utils');
const { checkParams, handleAsyncError } = require('../utils');
const bodySchema = Joi.object({
slug: Joi.string().required(),
exportFormat: Joi.string().valid('csv', 'json', 'json-v2').required(),
search: Joi.string().default(''),
applySearch: Joi.boolean().default(false),
relationsAsId: Joi.boolean().default(false),
deepness: Joi.number().integer().min(1).default(5),
});
const exportData = async (ctx) => {
let { slug, search, applySearch, exportFormat, relationsAsId, deepness } = checkParams(bodySchema, ctx.request.body);
let data;
if (exportFormat === getService('export').formats.JSON_V2) {
data = await getService('export').exportDataV2({ slug, search, applySearch, deepness });
} else {
data = await getService('export').exportData({ slug, search, applySearch, exportFormat, relationsAsId, deepness });
}
ctx.body = {
data,
};
};
module.exports = ({ strapi }) => ({
exportData: handleAsyncError(exportData),
});

View File

@ -0,0 +1 @@
module.exports = require('./export-controller');

View File

@ -0,0 +1,7 @@
'use strict';
const importData = require('./import-data');
module.exports = ({ strapi }) => ({
importData: importData({ strapi }),
});

View File

@ -0,0 +1,43 @@
'use strict';
const Joi = require('joi');
const { getService } = require('../../../utils');
const { checkParams, handleAsyncError } = require('../utils');
const bodySchema = Joi.object({
slug: Joi.string().required(),
data: Joi.any().required(),
format: Joi.string().valid('csv', 'json').required(),
idField: Joi.string(),
});
const importData = async (ctx) => {
const { user } = ctx.state;
const { slug, data: dataRaw, format, idField } = checkParams(bodySchema, ctx.request.body);
const fileContent = await getService('import').parseInputData(format, dataRaw, { slug });
let res;
if (fileContent?.version === 2) {
res = await getService('import').importDataV2(fileContent, {
slug,
user,
idField,
});
} else {
res = await getService('import').importData(dataRaw, {
slug,
format,
user,
idField,
});
}
ctx.body = {
failures: res.failures,
};
};
module.exports = ({ strapi }) => handleAsyncError(importData);

View File

@ -0,0 +1 @@
module.exports = require('./import-controller');

View File

@ -0,0 +1,49 @@
const { BusinessError } = require('../../utils/errors');
/**
* Check an object's properties based on a validation schema
* @param {*} schema - Validation schema
* @param {*} obj - Object to check properties
* @param {Object} options
* @param {boolean} [options.allowUnknown] - Whether to allow unknown properties (default: false)
* @returns Object with values checked and parsed
*/
const checkParams = (schema, obj, options = {}) => {
const allowUnknown = options.allowUnknown || false;
const validation = schema.validate(obj, {
abortEarly: false,
allowUnknown,
});
if (validation.error) {
const error = validation.error.details.map((detail) => detail.message).join(', ');
throw new BusinessError(error);
}
return validation.value;
};
const handleAsyncError = (fn) => async (ctx) => {
try {
const res = await fn(ctx);
return res;
} catch (err) {
strapi.log.error(err);
if (err instanceof BusinessError) {
ctx.status = 400;
ctx.body = {
message: err.message,
code: err.code,
};
} else {
throw err;
}
}
};
module.exports = {
checkParams,
handleAsyncError,
};

View File

@ -0,0 +1,13 @@
'use strict';
const exportAdminController = require('./admin/export-controller');
const importAdminController = require('./admin/import-controller');
const exportContentApiController = require('./content-api/export-controller');
const importContentApiController = require('./content-api/import-controller');
module.exports = {
exportAdmin: exportAdminController,
importAdmin: importAdminController,
export: exportContentApiController,
import: importContentApiController,
};

3
src/server/destroy.js Normal file
View File

@ -0,0 +1,3 @@
'use strict';
module.exports = ({ strapi }) => {};

25
src/server/index.js Normal file
View File

@ -0,0 +1,25 @@
'use strict';
const register = require('./register');
const bootstrap = require('./bootstrap');
const destroy = require('./destroy');
const config = require('./config');
const contentTypes = require('./content-types');
const controllers = require('./controllers');
const routes = require('./routes');
const middlewares = require('./middlewares');
const policies = require('./policies');
const services = require('./services');
module.exports = {
register,
bootstrap,
destroy,
config,
controllers,
routes,
services,
contentTypes,
policies,
middlewares,
};

View File

@ -0,0 +1,3 @@
'use strict';
module.exports = {};

View File

@ -0,0 +1,3 @@
'use strict';
module.exports = {};

5
src/server/register.js Normal file
View File

@ -0,0 +1,5 @@
'use strict';
module.exports = ({ strapi }) => {
// registeration phase
};

View File

@ -0,0 +1,13 @@
module.exports = {
type: 'admin',
routes: [
{
method: 'POST',
path: '/export/contentTypes',
handler: 'exportAdmin.exportData',
config: {
policies: [],
},
},
],
};

View File

@ -0,0 +1,13 @@
module.exports = {
type: 'content-api',
routes: [
{
method: 'POST',
path: '/content/export/contentTypes',
handler: 'export.exportData',
config: {
policies: [],
},
},
],
};

View File

@ -0,0 +1,21 @@
module.exports = {
type: 'admin',
routes: [
{
method: 'GET',
path: '/import/model-attributes/:slug',
handler: 'importAdmin.getModelAttributes',
config: {
policies: [],
},
},
{
method: 'POST',
path: '/import',
handler: 'importAdmin.importData',
config: {
policies: [],
},
},
],
};

View File

@ -0,0 +1,13 @@
module.exports = {
type: 'content-api',
routes: [
{
method: 'POST',
path: '/content/import',
handler: 'import.importData',
config: {
policies: [],
},
},
],
};

View File

@ -0,0 +1,11 @@
const exportAdminRoutes = require('./export-admin');
const importAdminRoutes = require('./import-admin');
const exportContentApiRoutes = require('./export-content-api');
const importContentApiRoutes = require('./import-content-api');
module.exports = {
exportAdminRoutes,
importAdminRoutes,
export: exportContentApiRoutes,
import: importContentApiRoutes,
};

View File

@ -0,0 +1,75 @@
const fromPairs = require('lodash/fromPairs');
const pick = require('lodash/pick');
const toPairs = require('lodash/toPairs');
const { CustomSlugToSlug, CustomSlugs } = require('../../config/constants');
const { getConfig } = require('../../utils/getConfig');
const convertToJson = (jsoContent) => {
return JSON.stringify(jsoContent, null, '\t');
};
const withBeforeConvert = (convertFn) => (jsoContent, options) => {
jsoContent = beforeConvert(jsoContent, options);
jsoContent = convertFn(jsoContent, options);
return jsoContent;
};
const beforeConvert = (jsoContent, options) => {
jsoContent = preprocess(jsoContent, options);
jsoContent = postprocess(jsoContent, options);
return jsoContent;
};
const preprocess = (jsoContent) => {
let media = jsoContent.data[CustomSlugToSlug[CustomSlugs.MEDIA]];
if (!media) {
return jsoContent;
}
media = fromPairs(
toPairs(media).map(([id, medium]) => {
if (isRelativeUrl(medium.url)) {
medium.url = buildAbsoluteUrl(medium.url);
}
return [id, medium];
}),
);
jsoContent.data[CustomSlugToSlug[CustomSlugs.MEDIA]] = media;
return jsoContent;
};
const isRelativeUrl = (url) => {
return url.startsWith('/');
};
const buildAbsoluteUrl = (relativeUrl) => {
return getConfig('serverPublicHostname') + relativeUrl;
};
const postprocess = (jsoContent) => {
let mediaSlug = CustomSlugToSlug[CustomSlugs.MEDIA];
let media = jsoContent.data[mediaSlug];
if (!media) {
return jsoContent;
}
media = fromPairs(
toPairs(media).map(([id, medium]) => {
medium = pick(medium, ['id', 'name', 'alternativeText', 'caption', 'hash', 'ext', 'mime', 'url', 'createdAt', 'updatedAt', 'createdBy', 'updatedBy']);
return [id, medium];
}),
);
jsoContent.data[mediaSlug] = media;
return jsoContent;
};
module.exports = {
convertToJson: withBeforeConvert(convertToJson),
};

View File

@ -0,0 +1,145 @@
const { isArraySafe, toArray } = require('../../../libs/arrays');
const { isObjectSafe } = require('../../../libs/objects');
const { CustomSlugToSlug, CustomSlugs } = require('../../config/constants');
const { getConfig } = require('../../utils/getConfig');
const { getModelAttributes, getModel } = require('../../utils/models');
const convertToCsv = (entries, options) => {
entries = toArray(entries);
const columnTitles = ['id'].concat(getModelAttributes(options.slug, { filterOutTarget: ['admin::user'] }).map((attr) => attr.name));
const content = [convertStrArrayToCsv(columnTitles)].concat(entries.map((entry) => convertEntryToStrArray(entry, columnTitles)).map(convertStrArrayToCsv)).join('\r\n');
return content;
};
const convertStrArrayToCsv = (entry) => {
return entry
.map(stringifyEntry)
.map((v) => v.replace(/"/g, '""'))
.map((v) => `"${v}"`)
.join(',');
};
const stringifyEntry = (entry) => {
if (typeof entry === 'object') {
return JSON.stringify(entry);
}
return String(entry);
};
const convertEntryToStrArray = (entry, keys) => {
return keys.map((key) => entry[key]);
};
const convertToJson = (entries, options) => {
entries = JSON.stringify(entries, null, '\t');
return entries;
};
const withBeforeConvert = (convertFn) => (entries, options) => {
entries = beforeConvert(entries, options);
entries = convertFn(entries, options);
return entries;
};
const beforeConvert = (entries, options) => {
entries = toArray(entries);
entries = exportMedia(entries, options);
if (options.relationsAsId) {
entries = exportRelationsAsId(entries, options);
}
if (getModel(options.slug).kind === 'singleType') {
return entries?.[0];
}
return entries;
};
const exportMedia = (entries, options) => {
if (options.slug === CustomSlugToSlug[CustomSlugs.MEDIA]) {
entries = entries.map((entry) => {
if (isObjectSafe(entry) && entry.url.startsWith('/')) {
entry.url = computeUrl(entry.url);
}
return entry;
});
return entries;
}
const mediaKeys = getModelAttributes(options.slug, { filterOutTarget: ['admin::user'], filterType: ['media'] }).map((attr) => attr.name);
const relationsAttr = getModelAttributes(options.slug, { filterOutTarget: ['admin::user'], filterType: ['component', 'dynamiczone', 'relation'] });
entries = entries.map((entry) => {
mediaKeys.forEach((key) => {
if (isArraySafe(entry[key])) {
entry[key] = entry[key].map((entryItem) => {
if (isObjectSafe(entryItem) && entryItem.url.startsWith('/')) {
entryItem.url = computeUrl(entryItem.url);
}
return entryItem;
});
} else if (isObjectSafe(entry[key]) && entry[key].url.startsWith('/')) {
entry[key].url = computeUrl(entry[key].url);
}
});
relationsAttr.forEach((attr) => {
if (attr.type === 'component') {
if (entry[attr.name]) {
const areMultiple = attr.repeatable;
const relEntriesProcessed = exportMedia(toArray(entry[attr.name]), { slug: attr.component });
entry[attr.name] = areMultiple ? relEntriesProcessed : relEntriesProcessed?.[0] || null;
}
} else if (attr.type === 'dynamiczone') {
if (entry[attr.name]) {
entry[attr.name] = entry[attr.name].map((component) => exportMedia([component], { slug: component.__component })?.[0] || null);
}
} else if (attr.type === 'relation') {
if (entry[attr.name]) {
const areMultiple = isArraySafe(entry[attr.name]);
const relEntriesProcessed = exportMedia(toArray(entry[attr.name]), { slug: attr.target });
entry[attr.name] = areMultiple ? relEntriesProcessed : relEntriesProcessed?.[0] || null;
}
}
});
return entry;
});
return entries;
};
const computeUrl = (relativeUrl) => {
return getConfig('serverPublicHostname') + relativeUrl;
};
const exportRelationsAsId = (entries, options) => {
const relationKeys = getModelAttributes(options.slug, { filterOutTarget: ['admin::user'], filterType: ['component', 'dynamiczone', 'media', 'relation'] }).map(
(attr) => attr.name,
);
return entries.map((entry) => {
relationKeys.forEach((key) => {
if (entry[key] == null) {
entry[key] = null;
} else if (isArraySafe(entry[key])) {
entry[key] = entry[key].map((rel) => {
if (typeof rel === 'object') {
return rel.id;
}
return rel;
});
} else if (isObjectSafe(entry[key])) {
entry[key] = entry[key].id;
}
});
return entry;
});
};
module.exports = {
convertToCsv: withBeforeConvert(convertToCsv),
convertToJson: withBeforeConvert(convertToJson),
};

View File

@ -0,0 +1,390 @@
const cloneDeep = require('lodash/cloneDeep');
const flattenDeep = require('lodash/flattenDeep');
const fromPairs = require('lodash/fromPairs');
const { isEmpty, merge } = require('lodash/fp');
const qs = require('qs');
const { isArraySafe, toArray } = require('../../../libs/arrays');
const { ObjectBuilder, isObjectSafe, mergeObjects } = require('../../../libs/objects');
const { CustomSlugToSlug, CustomSlugs } = require('../../config/constants');
const { getModelAttributes, getAllSlugs } = require('../../utils/models');
const { convertToJson } = require('./converters-v2');
const dataFormats = {
JSON: 'json',
};
const dataConverterConfigs = {
[dataFormats.JSON]: {
convertEntries: convertToJson,
},
};
/**
* Export data.
* @param {Object} options
* @param {string} options.slug
* @param {string} options.search
* @param {boolean} options.applySearch
* @param {boolean} options.relationsAsId
* @param {number} options.deepness
* @returns {string}
*/
const exportDataV2 = async ({ slug, search, applySearch, deepness = 5 }) => {
slug = CustomSlugToSlug[slug] || slug;
let entries = {};
if (slug === CustomSlugs.WHOLE_DB) {
for (const slug of getAllSlugs()) {
const hierarchy = buildSlugHierarchy(slug, deepness);
const slugEntries = await findEntriesForHierarchy(slug, hierarchy, deepness, { ...(applySearch ? { search } : {}) });
entries = mergeObjects(entries, slugEntries);
}
} else {
const hierarchy = buildSlugHierarchy(slug, deepness);
entries = await findEntriesForHierarchy(slug, hierarchy, deepness, { ...(applySearch ? { search } : {}) });
}
const jsoContent = {
version: 2,
data: entries,
};
const fileContent = convertData(jsoContent, {
slug,
dataFormat: 'json',
});
return fileContent;
};
const findEntriesForHierarchy = async (slug, hierarchy, deepness, { search, ids }) => {
let storedData = {};
if (slug === 'admin::user') {
return storedData;
}
let entries = await findEntries(slug, deepness, { search, ids })
.then((entries) => {
entries = toArray(entries).filter(Boolean);
const isModelLocalized = !!hierarchy?.localizations;
// Export locales
if (isModelLocalized) {
const allEntries = [...entries];
const entryIdsToExported = fromPairs(allEntries.map((entry) => [entry.id, true]));
for (const entry of entries) {
entry.localizations.forEach((locale) => {
if (!entryIdsToExported[locale.id]) {
allEntries.push(locale);
entryIdsToExported[locale.id] = true;
}
});
}
return allEntries;
}
return entries;
})
.then((entries) => toArray(entries));
// Transform relations as ids.
let entriesFlatten = cloneDeep(entries);
(() => {
const flattenEntryDynamicZone = (dynamicZoneEntries) => {
if (isArraySafe(dynamicZoneEntries)) {
return dynamicZoneEntries.map((entry) => ({
__component: entry.__component,
id: entry.id,
}));
}
return dynamicZoneEntries;
};
const flattenEntryCommon = (entry) => {
if (entry == null) {
return null;
} else if (isArraySafe(entry)) {
return entry.map((rel) => {
if (typeof rel === 'object') {
return rel.id;
}
return rel;
});
} else if (isObjectSafe(entry)) {
return entry.id;
}
return entry;
};
const flattenEntry = (entry, slug) => {
const attributes = getModelAttributes(slug, { filterType: ['component', 'dynamiczone', 'media', 'relation'] });
attributes.forEach((attribute) => {
const flattener = attribute.type === 'dynamiczone' ? flattenEntryDynamicZone : flattenEntryCommon;
entry[attribute.name] = flattener(entry[attribute.name]);
});
return entry;
};
entriesFlatten = entriesFlatten.map((entry) => flattenEntry(entry, slug));
})();
storedData = mergeObjects({ [slug]: Object.fromEntries(entriesFlatten.map((entry) => [entry.id, entry])) }, storedData);
// Skip admin::user slug.
(() => {
const relations = getModelAttributes(slug, { filterType: ['relation'] });
return entries.map((entry) => {
relations.forEach((relation) => {
if (relation.target === 'admin::user') {
delete entry[relation.name];
}
});
return entry;
});
})();
await (async () => {
let attributes = getModelAttributes(slug, { filterType: ['component'] });
for (const attribute of attributes) {
if (!hierarchy[attribute.name]?.__slug) {
continue;
}
const ids = flattenDeep(
entries
.filter((entry) => !!entry[attribute.name])
.map((entry) => entry[attribute.name])
.map(toArray)
.map((entryArray) => entryArray.filter((entry) => !!entry.id).map((entry) => entry.id)),
);
const subStore = await findEntriesForHierarchy(hierarchy[attribute.name].__slug, hierarchy[attribute.name], deepness - 1, { ids });
storedData = mergeObjects(subStore, storedData);
}
})();
await (async () => {
let attributes = getModelAttributes(slug, { filterType: ['dynamiczone'] });
for (const attribute of attributes) {
for (const componentSlug of attribute.components) {
const componentHierarchy = hierarchy[attribute.name]?.[componentSlug];
if (!componentHierarchy?.__slug) {
continue;
}
const componentEntries = flattenDeep(
entries
.map((entry) => {
return entry;
})
.filter((entry) => !!entry[attribute.name])
.map((entry) => entry[attribute.name]),
).filter((entry) => entry?.__component === componentSlug);
const ids = componentEntries.map((entry) => entry.id);
const subStore = await findEntriesForHierarchy(componentHierarchy.__slug, componentHierarchy, deepness - 1, { ids });
storedData = mergeObjects(subStore, storedData);
}
}
})();
await (async () => {
let attributes = getModelAttributes(slug, { filterType: ['media'] });
for (const attribute of attributes) {
if (!hierarchy[attribute.name]?.__slug) {
continue;
}
const ids = flattenDeep(
entries
.filter((entry) => !!entry[attribute.name])
.map((entry) => entry[attribute.name])
.map(toArray)
.map((entryArray) => entryArray.filter((entry) => !!entry.id).map((entry) => entry.id)),
);
const subStore = await findEntriesForHierarchy(hierarchy[attribute.name].__slug, hierarchy[attribute.name], deepness - 1, { ids });
storedData = mergeObjects(subStore, storedData);
}
})();
await (async () => {
let attributes = getModelAttributes(slug, { filterType: ['relation'] });
for (const attribute of attributes) {
if (!hierarchy[attribute.name]?.__slug) {
continue;
}
const ids = flattenDeep(
entries
.filter((entry) => !!entry[attribute.name])
.map((entry) => entry[attribute.name])
.map(toArray)
.map((entryArray) => entryArray.filter((entry) => !!entry.id).map((entry) => entry.id)),
);
const subStore = await findEntriesForHierarchy(hierarchy[attribute.name].__slug, hierarchy[attribute.name], deepness - 1, { ids });
storedData = mergeObjects(subStore, storedData);
}
})();
return storedData;
};
const findEntries = async (slug, deepness, { search, ids }) => {
try {
const queryBuilder = new ObjectBuilder();
queryBuilder.extend(getPopulateFromSchema(slug, deepness));
if (search) {
queryBuilder.extend(buildFilterQuery(search));
} else if (ids) {
queryBuilder.extend({
filters: {
id: { $in: ids },
},
});
}
const entries = await strapi.entityService.findMany(slug, queryBuilder.get());
return entries;
} catch (_) {
return [];
}
};
const buildFilterQuery = (search) => {
let { filters, sort: sortRaw } = qs.parse(search);
const [attr, value] = sortRaw?.split(':') || [];
let sort = {};
if (attr && value) {
sort[attr] = value.toLowerCase();
}
return {
filters,
sort,
};
};
/**
*
* @param {Object} data
* @param {Array<Object>} data.entries
* @param {Record<string, any>} data.hierarchy
* @param {Object} options
* @param {string} options.slug
* @param {string} options.dataFormat
* @param {boolean} options.relationsAsId
* @returns
*/
const convertData = (data, options) => {
const converter = getConverter(options.dataFormat);
const convertedData = converter.convertEntries(data, options);
return convertedData;
};
const getConverter = (dataFormat) => {
const converter = dataConverterConfigs[dataFormat];
if (!converter) {
throw new Error(`Data format ${dataFormat} is not supported.`);
}
return converter;
};
const getPopulateFromSchema = (slug, deepness = 5) => {
if (deepness <= 1) {
return true;
}
if (slug === 'admin::user') {
return undefined;
}
const populate = {};
const model = strapi.getModel(slug);
for (const [attributeName, attribute] of Object.entries(getModelPopulationAttributes(model))) {
if (!attribute) {
continue;
}
if (attribute.type === 'component') {
populate[attributeName] = getPopulateFromSchema(attribute.component, deepness - 1);
} else if (attribute.type === 'dynamiczone') {
const dynamicPopulate = attribute.components.reduce((zonePopulate, component) => {
const compPopulate = getPopulateFromSchema(component, deepness - 1);
return compPopulate === true ? zonePopulate : merge(zonePopulate, compPopulate);
}, {});
populate[attributeName] = isEmpty(dynamicPopulate) ? true : dynamicPopulate;
} else if (attribute.type === 'relation') {
const relationPopulate = getPopulateFromSchema(attribute.target, deepness - 1);
if (relationPopulate) {
populate[attributeName] = relationPopulate;
}
} else if (attribute.type === 'media') {
populate[attributeName] = true;
}
}
return isEmpty(populate) ? true : { populate };
};
const buildSlugHierarchy = (slug, deepness = 5) => {
slug = CustomSlugToSlug[slug] || slug;
if (deepness <= 1) {
return { __slug: slug };
}
const hierarchy = {
__slug: slug,
};
const model = strapi.getModel(slug);
for (const [attributeName, attribute] of Object.entries(getModelPopulationAttributes(model))) {
if (!attribute) {
continue;
}
if (attribute.type === 'component') {
hierarchy[attributeName] = buildSlugHierarchy(attribute.component, deepness - 1);
} else if (attribute.type === 'dynamiczone') {
hierarchy[attributeName] = Object.fromEntries(attribute.components.map((componentSlug) => [componentSlug, buildSlugHierarchy(componentSlug, deepness - 1)]));
} else if (attribute.type === 'relation') {
const relationHierarchy = buildSlugHierarchy(attribute.target, deepness - 1);
if (relationHierarchy) {
hierarchy[attributeName] = relationHierarchy;
}
} else if (attribute.type === 'media') {
hierarchy[attributeName] = buildSlugHierarchy(CustomSlugs.MEDIA, deepness - 1);
}
}
return hierarchy;
};
const getModelPopulationAttributes = (model) => {
if (model.uid === 'plugin::upload.file') {
const { related, ...attributes } = model.attributes;
return attributes;
}
return model.attributes;
};
module.exports = {
exportDataV2,
};

View File

@ -0,0 +1,147 @@
const { isEmpty, merge } = require('lodash/fp');
const qs = require('qs');
const { ObjectBuilder } = require('../../../libs/objects');
const { CustomSlugToSlug } = require('../../config/constants');
const { convertToCsv, convertToJson } = require('./converters');
const dataFormats = {
CSV: 'csv',
JSON: 'json',
JSON_V2: 'json-v2',
};
const dataConverterConfigs = {
[dataFormats.CSV]: {
convertEntries: convertToCsv,
},
[dataFormats.JSON]: {
convertEntries: convertToJson,
},
};
/**
* Export data.
* @param {Object} options
* @param {string} options.slug
* @param {("csv"|"json")} options.exportFormat
* @param {string} options.search
* @param {boolean} options.applySearch
* @param {boolean} options.relationsAsId
* @param {number} options.deepness
* @returns {string}
*/
const exportData = async ({ slug, search, applySearch, exportFormat, relationsAsId, deepness = 5 }) => {
const slugToProcess = CustomSlugToSlug[slug] || slug;
const queryBuilder = new ObjectBuilder();
queryBuilder.extend(getPopulateFromSchema(slugToProcess, deepness));
if (applySearch) {
queryBuilder.extend(buildFilterQuery(search));
}
const query = queryBuilder.get();
const entries = await strapi.entityService.findMany(slugToProcess, query);
const data = convertData(entries, {
slug: slugToProcess,
dataFormat: exportFormat,
relationsAsId,
});
return data;
};
const buildFilterQuery = (search) => {
let { filters, sort: sortRaw } = qs.parse(search);
const [attr, value] = sortRaw?.split(':') || [];
let sort = {};
if (attr && value) {
sort[attr] = value.toLowerCase();
}
return {
filters,
sort,
};
};
/**
*
* @param {Array<Object>} entries
* @param {Object} options
* @param {string} options.slug
* @param {string} options.dataFormat
* @param {boolean} options.relationsAsId
* @returns
*/
const convertData = (entries, options) => {
const converter = getConverter(options.dataFormat);
const convertedData = converter.convertEntries(entries, options);
return convertedData;
};
const getConverter = (dataFormat) => {
const converter = dataConverterConfigs[dataFormat];
if (!converter) {
throw new Error(`Data format ${dataFormat} is not supported.`);
}
return converter;
};
const getPopulateFromSchema = (slug, deepness = 5) => {
if (deepness <= 1) {
return true;
}
if (slug === 'admin::user') {
return undefined;
}
const populate = {};
const model = strapi.getModel(slug);
for (const [attributeName, attribute] of Object.entries(getModelPopulationAttributes(model))) {
if (!attribute) {
continue;
}
if (attribute.type === 'component') {
populate[attributeName] = getPopulateFromSchema(attribute.component, deepness - 1);
} else if (attribute.type === 'dynamiczone') {
const dynamicPopulate = attribute.components.reduce((zonePopulate, component) => {
const compPopulate = getPopulateFromSchema(component, deepness - 1);
return compPopulate === true ? zonePopulate : merge(zonePopulate, compPopulate);
}, {});
populate[attributeName] = isEmpty(dynamicPopulate) ? true : dynamicPopulate;
} else if (attribute.type === 'relation') {
const relationPopulate = getPopulateFromSchema(attribute.target, deepness - 1);
if (relationPopulate) {
populate[attributeName] = relationPopulate;
}
} else if (attribute.type === 'media') {
populate[attributeName] = true;
}
}
return isEmpty(populate) ? true : { populate };
};
const getModelPopulationAttributes = (model) => {
if (model.uid === 'plugin::upload.file') {
const { related, ...attributes } = model.attributes;
return attributes;
}
return model.attributes;
};
module.exports = {
formats: dataFormats,
exportData,
getPopulateFromSchema,
};

View File

@ -0,0 +1,9 @@
const { formats, exportData, getPopulateFromSchema } = require('./export');
const { exportDataV2 } = require('./export-v2');
module.exports = {
formats,
exportData,
getPopulateFromSchema,
exportDataV2,
};

View File

@ -0,0 +1,327 @@
const cloneDeep = require('lodash/cloneDeep');
const isEmpty = require('lodash/isEmpty');
const omit = require('lodash/omit');
const pick = require('lodash/pick');
const { isArraySafe } = require('../../../libs/arrays');
const { ObjectBuilder } = require('../../../libs/objects');
const { CustomSlugs, CustomSlugToSlug } = require('../../config/constants');
const { getModel, getModelAttributes, getModelConfig } = require('../../utils/models');
const { findOrImportFile } = require('./utils/file');
/**
* @typedef {Object} ImportDataRes
* @property {Array<ImportDataFailures>} failures
*/
/**
* Represents failed imports.
* @typedef {Object} ImportDataFailures
* @property {Error} error - Error raised.
* @property {Object} data - Data for which import failed.
*/
/**
* Import data.
* @param {Object} fileContent - Content of the import file.
* @param {Object} options
* @param {string} options.slug - Slug of the imported model.
* @param {Object} options.user - User importing the data.
* @param {Object} options.idField - Field used as unique identifier.
* @returns {Promise<ImportDataRes>}
*/
const importDataV2 = async (fileContent, { slug, user, idField }) => {
const { data } = fileContent;
const slugs = Object.keys(data);
let failures = [];
// Import without setting relations.
for (const slugFromFile of slugs) {
let slugFailures = [];
if (slugFromFile === CustomSlugToSlug[CustomSlugs.MEDIA]) {
slugFailures = await importMedia(Object.values(data[slugFromFile]), { user }).then((res) => res.slugFailures);
} else {
slugFailures = await importOtherSlug(Object.values(data[slugFromFile]), {
slug: slugFromFile,
user,
// Keep behavior of `idField` of version 1.
...(slugFromFile === slug ? { idField } : {}),
importStage: 'simpleAttributes',
}).then((res) => res.failures);
}
failures = [...failures, ...(slugFailures || [])];
}
// Set relations relations.
for (const slugFromFile of slugs) {
let slugFailures = [];
if (slugFromFile === CustomSlugToSlug[CustomSlugs.MEDIA]) {
slugFailures = await importMedia(Object.values(data[slugFromFile]), { user }).then((res) => res.slugFailures);
} else {
slugFailures = await importOtherSlug(Object.values(data[slugFromFile]), {
slug: slugFromFile,
user,
// Keep behavior of `idField` of version 1.
...(slugFromFile === slug ? { idField } : {}),
importStage: 'relationAttributes',
}).then((res) => res.failures);
}
failures = [...failures, ...(slugFailures || [])];
}
// Sync primary key sequence for postgres databases.
// See https://github.com/strapi/strapi/issues/12493.
if (strapi.db.config.connection.client === 'postgres') {
for (const slugFromFile of slugs) {
const model = getModel(slugFromFile);
await strapi.db.connection.raw(`SELECT SETVAL((SELECT PG_GET_SERIAL_SEQUENCE('${model.collectionName}', 'id')), (SELECT MAX(id) FROM ${model.collectionName}) + 1, FALSE);`);
}
}
return { failures };
};
const importMedia = async (fileData, { user }) => {
const processed = [];
for (let fileDatum of fileData) {
let res;
try {
await findOrImportFile(fileDatum, user, { allowedFileTypes: ['any'] });
res = { success: true };
} catch (err) {
strapi.log.error(err);
res = { success: false, error: err.message, args: [fileDatum] };
}
processed.push(res);
}
const failures = processed.filter((p) => !p.success).map((f) => ({ error: f.error, data: f.args[0] }));
return {
failures,
};
};
/**
* Import data.
* @param {Array<Object>} data
* @param {Object} importOptions
* @param {('simpleAttributes'|'relationAttributes')} [importOptions.importStage]
*/
const importOtherSlug = async (data, { slug, user, idField, importStage }) => {
// Sort localized data with default locale first.
await (async () => {
const { isLocalized } = getModelConfig(slug);
if (isLocalized) {
const defaultLocale = await strapi.plugin('i18n').service('locales').getDefaultLocale();
data = data.sort((dataA, dataB) => {
if (dataA?.locale === defaultLocale && dataB?.locale === defaultLocale) {
return 0;
} else if (dataA?.locale === defaultLocale) {
return -1;
}
return 1;
});
}
})();
const processed = [];
for (let datum of data) {
let res;
try {
await updateOrCreate(user, slug, datum, idField, { importStage });
res = { success: true };
} catch (err) {
strapi.log.error(err);
res = { success: false, error: err.message, args: [datum] };
}
processed.push(res);
}
const failures = processed.filter((p) => !p.success).map((f) => ({ error: f.error, data: f.args[0] }));
return {
failures,
};
};
/**
* Update or create entries for a given model.
* @param {Object} user - User importing the data.
* @param {string} slug - Slug of the model.
* @param {Object} datum - Data to update/create entries from.
* @param {string} idField - Field used as unique identifier.
* @param {Object} importOptions
* @param {('simpleAttributes'|'relationAttributes')} [importOptions.importStage]
*/
const updateOrCreate = async (user, slug, datum, idField = 'id', { importStage }) => {
datum = cloneDeep(datum);
if (importStage == 'simpleAttributes') {
const attributeNames = getModelAttributes(slug, { filterOutType: ['component', 'dynamiczone', 'media', 'relation'], addIdAttribute: true })
.map(({ name }) => name)
.concat('localizations', 'locale');
datum = pick(datum, attributeNames);
} else if (importStage === 'relationAttributes') {
const attributeNames = getModelAttributes(slug, { filterType: ['component', 'dynamiczone', 'media', 'relation'], addIdAttribute: true })
.map(({ name }) => name)
.concat('localizations', 'locale');
datum = pick(datum, attributeNames);
}
const model = getModel(slug);
if (model.kind === 'singleType') {
await updateOrCreateSingleType(user, slug, datum, { importStage });
} else {
await updateOrCreateCollectionType(user, slug, datum, { idField, importStage });
}
};
/**
* Update or create entries for a given model.
* @param {Object} user - User importing the data.
* @param {string} slug - Slug of the model.
* @param {Object} datum - Data to update/create entries from.
* @param {Object} importOptions
* @param {string} [importOptions.idField] - Field used as unique identifier.
* @param {('simpleAttributes'|'relationAttributes')} [importOptions.importStage]
*/
const updateOrCreateCollectionType = async (user, slug, datum, { idField, importStage }) => {
const { isLocalized } = getModelConfig(slug);
const whereBuilder = new ObjectBuilder();
if (datum[idField]) {
whereBuilder.extend({ [idField]: datum[idField] });
}
const where = whereBuilder.get();
if (!isLocalized) {
let entry = await strapi.db.query(slug).findOne({ where });
if (!entry) {
await strapi.entityService.create(slug, { data: datum });
} else {
await updateEntry(slug, entry.id, datum, { importStage });
}
} else {
if (!datum.locale) {
throw new Error(`No locale set to import entry for slug ${slug} (data ${JSON.stringify(datum)})`);
}
const defaultLocale = await strapi.plugin('i18n').service('locales').getDefaultLocale();
const isDatumInDefaultLocale = datum.locale === defaultLocale;
let entryDefaultLocale = null;
let entry = await strapi.db.query(slug).findOne({ where, populate: ['localizations'] });
if (isDatumInDefaultLocale) {
entryDefaultLocale = entry;
} else {
if (entry) {
// If `entry` has been found, `entry` holds the data for the default locale and
// the data for other locales in its `localizations` attribute.
const localizedEntries = [entry, ...(entry?.localizations || [])];
entryDefaultLocale = localizedEntries.find((e) => e.locale === defaultLocale);
entry = localizedEntries.find((e) => e.locale === datum.locale);
} else {
// Otherwise try to find entry for default locale through localized siblings.
let localizationIdx = 0;
const localizations = datum?.localizations || [];
while (localizationIdx < localizations.length && !entryDefaultLocale && !entry) {
const id = localizations[localizationIdx];
const localizedEntry = await strapi.db.query(slug).findOne({ where: { id }, populate: ['localizations'] });
const localizedEntries = localizedEntry != null ? [localizedEntry, ...(localizedEntry?.localizations || [])] : [];
if (!entryDefaultLocale) {
entryDefaultLocale = localizedEntries.find((e) => e.locale === defaultLocale);
}
if (!entry) {
entry = localizedEntries.find((e) => e.locale === datum.locale);
}
localizationIdx += 1;
}
}
}
datum = omit(datum, ['localizations']);
if (isEmpty(omit(datum, ['id']))) {
return;
}
if (isDatumInDefaultLocale) {
if (!entryDefaultLocale) {
await strapi.entityService.create(slug, { data: datum });
} else {
await strapi.entityService.update(slug, entryDefaultLocale.id, { data: omit({ ...datum }, ['id']) });
}
} else {
if (!entryDefaultLocale) {
throw new Error(`Could not find default locale entry to import localization for slug ${slug} (data ${JSON.stringify(datum)})`);
}
datum = omit({ ...datum }, ['id']);
if (!entry) {
const createHandler = strapi.plugin('i18n').service('core-api').createCreateLocalizationHandler(getModel(slug));
await createHandler({ id: entryDefaultLocale.id, data: datum });
} else {
await strapi.entityService.update(slug, entry.id, { data: datum });
}
}
}
};
const updateOrCreateSingleType = async (user, slug, datum, { importStage }) => {
const { isLocalized } = getModelConfig(slug);
if (!isLocalized) {
let entry = await strapi.db.query(slug).findMany();
entry = isArraySafe(entry) ? entry[0] : entry;
if (!entry) {
await strapi.entityService.create(slug, { data: datum });
} else {
await updateEntry(slug, entry.id, datum, { importStage });
}
} else {
const defaultLocale = await strapi.plugin('i18n').service('locales').getDefaultLocale();
const isDatumInDefaultLocale = !datum.locale || datum.locale === defaultLocale;
datum = omit(datum, ['localizations']);
if (isEmpty(omit(datum, ['id']))) {
return;
}
let entryDefaultLocale = await strapi.db.query(slug).findOne({ where: { locale: defaultLocale } });
if (!entryDefaultLocale) {
entryDefaultLocale = await strapi.entityService.create(slug, { data: { ...datum, locale: defaultLocale } });
}
if (isDatumInDefaultLocale) {
if (!entryDefaultLocale) {
await strapi.entityService.create(slug, { data: datum });
} else {
await strapi.entityService.update(slug, entryDefaultLocale.id, { data: datum });
}
} else {
const entryLocale = await strapi.db.query(slug).findOne({ where: { locale: datum.locale } });
let datumLocale = { ...entryLocale, ...datum };
await strapi.db.query(slug).delete({ where: { locale: datum.locale } });
const createHandler = strapi.plugin('i18n').service('core-api').createCreateLocalizationHandler(getModel(slug));
await createHandler({ id: entryDefaultLocale.id, data: datumLocale });
}
}
};
const updateEntry = async (slug, id, datum, { importStage }) => {
datum = omit(datum, ['id']);
if (importStage === 'simpleAttributes') {
await strapi.entityService.update(slug, id, { data: datum });
} else if (importStage === 'relationAttributes') {
await strapi.db.query(slug).update({ where: { id }, data: datum });
}
};
module.exports = {
importDataV2,
};

View File

@ -0,0 +1,216 @@
const { isArraySafe, toArray } = require('../../../libs/arrays');
const { ObjectBuilder, isObjectSafe } = require('../../../libs/objects');
const { CustomSlugs } = require('../../config/constants');
const { getModelAttributes, getModel } = require('../../utils/models');
const { findOrImportFile } = require('./utils/file');
const { parseInputData } = require('./parsers');
/**
* @typedef {Object} ImportDataRes
* @property {Array<ImportDataFailures>} failures
*/
/**
* Represents failed imports.
* @typedef {Object} ImportDataFailures
* @property {Error} error - Error raised.
* @property {Object} data - Data for which import failed.
*/
/**
* Import data.
* @param {Array<Object>} dataRaw - Data to import.
* @param {Object} options
* @param {string} options.slug - Slug of the model to import.
* @param {("csv" | "json")} options.format - Format of the imported data.
* @param {Object} options.user - User importing the data.
* @param {Object} options.idField - Field used as unique identifier.
* @returns {Promise<ImportDataRes>}
*/
const importData = async (dataRaw, { slug, format, user, idField }) => {
let data = await parseInputData(format, dataRaw, { slug });
data = toArray(data);
let res;
if (slug === CustomSlugs.MEDIA) {
res = await importMedia(data, { user });
} else {
res = await importOtherSlug(data, { slug, user, idField });
}
return res;
};
const importMedia = async (fileData, { user }) => {
const processed = [];
for (let fileDatum of fileData) {
let res;
try {
await findOrImportFile(fileDatum, user, { allowedFileTypes: ['any'] });
res = { success: true };
} catch (err) {
strapi.log.error(err);
res = { success: false, error: err.message, args: [fileDatum] };
}
processed.push(res);
}
const failures = processed.filter((p) => !p.success).map((f) => ({ error: f.error, data: f.args[0] }));
return {
failures,
};
};
const importOtherSlug = async (data, { slug, user, idField }) => {
const processed = [];
for (let datum of data) {
let res;
try {
await updateOrCreate(user, slug, datum, idField);
res = { success: true };
} catch (err) {
strapi.log.error(err);
res = { success: false, error: err.message, args: [datum] };
}
processed.push(res);
}
const failures = processed.filter((p) => !p.success).map((f) => ({ error: f.error, data: f.args[0] }));
return {
failures,
};
};
/**
* Update or create entries for a given model.
* @param {Object} user - User importing the data.
* @param {string} slug - Slug of the model.
* @param {Object} data - Data to update/create entries from.
* @param {string} idField - Field used as unique identifier.
* @returns Updated/created entry.
*/
const updateOrCreate = async (user, slug, data, idField = 'id') => {
const relationAttributes = getModelAttributes(slug, { filterType: ['component', 'dynamiczone', 'media', 'relation'] });
for (let attribute of relationAttributes) {
data[attribute.name] = await updateOrCreateRelation(user, attribute, data[attribute.name]);
}
let entry;
const model = getModel(slug);
if (model.kind === 'singleType') {
entry = await updateOrCreateSingleType(user, slug, data, idField);
} else {
entry = await updateOrCreateCollectionType(user, slug, data, idField);
}
return entry;
};
const updateOrCreateCollectionType = async (user, slug, data, idField) => {
const whereBuilder = new ObjectBuilder();
if (data[idField]) {
whereBuilder.extend({ [idField]: data[idField] });
}
const where = whereBuilder.get();
// Prevent strapi from throwing a unique constraint error on id field.
if (idField !== 'id') {
delete data.id;
}
let entry;
if (!where[idField]) {
entry = await strapi.db.query(slug).create({ data });
} else {
entry = await strapi.db.query(slug).update({ where, data });
if (!entry) {
entry = await strapi.db.query(slug).create({ data });
}
}
return entry;
};
const updateOrCreateSingleType = async (user, slug, data, idField) => {
delete data.id;
let [entry] = await strapi.db.query(slug).findMany();
if (!entry) {
entry = await strapi.db.query(slug).create({ data });
} else {
entry = await strapi.db.query(slug).update({ where: { id: entry.id }, data });
}
return entry;
};
/**
* Update or create a relation.
* @param {Object} user
* @param {Attribute} rel
* @param {number | Object | Array<Object>} relData
*/
const updateOrCreateRelation = async (user, rel, relData) => {
if (relData == null) {
return null;
}
if (['createdBy', 'updatedBy'].includes(rel.name)) {
return user.id;
} else if (rel.type === 'dynamiczone') {
const components = [];
for (const componentDatum of relData || []) {
let component = await updateOrCreate(user, componentDatum.__component, componentDatum);
component = { ...component, __component: componentDatum.__component };
components.push(component);
}
return components;
} else if (rel.type === 'component') {
relData = toArray(relData);
relData = rel.repeatable ? relData : relData.slice(0, 1);
const entryIds = [];
for (const relDatum of relData) {
if (typeof relDatum === 'number') {
entryIds.push(relDatum);
} else if (isObjectSafe(relDatum)) {
const entry = await updateOrCreate(user, rel.component, relDatum);
if (entry?.id) {
entryIds.push(entry.id);
}
}
}
return rel.repeatable ? entryIds : entryIds?.[0] || null;
} else if (rel.type === 'media') {
relData = toArray(relData);
relData = rel.multiple ? relData : relData.slice(0, 1);
const entryIds = [];
for (const relDatum of relData) {
const media = await findOrImportFile(relDatum, user, { allowedFileTypes: rel.allowedTypes });
if (media?.id) {
entryIds.push(media.id);
}
}
return rel.multiple ? entryIds : entryIds?.[0] || null;
} else if (rel.type === 'relation') {
const isMultiple = isArraySafe(relData);
relData = toArray(relData);
const entryIds = [];
for (const relDatum of relData) {
if (typeof relDatum === 'number') {
entryIds.push(relDatum);
} else if (isObjectSafe(relDatum)) {
const entry = await updateOrCreate(user, rel.target, relDatum);
if (entry?.id) {
entryIds.push(entry.id);
}
}
}
return isMultiple ? entryIds : entryIds?.[0] || null;
}
throw new Error(`Could not update or create relation of type ${rel.type}.`);
};
module.exports = {
importData,
};

View File

@ -0,0 +1,9 @@
const { importData } = require('./import');
const { importDataV2 } = require('./import-v2');
const { parseInputData } = require('./parsers');
module.exports = {
importData,
importDataV2,
parseInputData,
};

View File

@ -0,0 +1,65 @@
'use strict';
const csvtojson = require('csvtojson');
const { isArraySafe } = require('../../../libs/arrays');
const { isObjectSafe } = require('../../../libs/objects');
const { getModelAttributes } = require('../../utils/models');
const parseCsv = async (dataRaw, { slug }) => {
let data = await csvtojson().fromString(dataRaw);
const relationNames = getModelAttributes(slug, { filterType: ['component', 'dynamiczone', 'media', 'relation'] }).map((a) => a.name);
data = data.map((datum) => {
for (let name of relationNames) {
try {
datum[name] = JSON.parse(datum[name]);
} catch (err) {
strapi.log.error(err);
}
}
return datum;
});
return data;
};
const parseJson = async (dataRaw) => {
let data = JSON.parse(dataRaw);
return data;
};
const parseJso = async (dataRaw) => {
if (!isObjectSafe(dataRaw) && !isArraySafe(dataRaw)) {
throw new Error(`To import JSO, data must be an array or an object`);
}
return dataRaw;
};
const parsers = {
csv: parseCsv,
jso: parseJso,
json: parseJson,
};
/**
* Parse input data.
* @param {("csv"|"jso"|"json")} format
* @param {*} dataRaw
* @param {Object} options
* @param {string} options.slug
*/
const parseInputData = async (format, dataRaw, { slug }) => {
const parser = parsers[format];
if (!parser) {
throw new Error(`Data input format ${format} is not supported.`);
}
const data = await parser(dataRaw, { slug });
return data;
};
module.exports = {
parseInputData,
};

View File

@ -0,0 +1,226 @@
const fs = require('fs');
const fse = require('fs-extra');
const last = require('lodash/last');
const trim = require('lodash/trim');
const os = require('os');
const path = require('path');
const fetch = require('node-fetch');
const { isObjectSafe } = require('../../../../libs/objects');
/**
* Find or import a file.
* @param {*} fileData - Strapi file data.
* @param {*} user - Strapi user.
* @param {Object} options
* @param {Array<string>} options.allowedFileTypes - File types the file should match (see Strapi file allowedTypes).
* @returns
*/
const findOrImportFile = async (fileData, user, { allowedFileTypes }) => {
let obj = {};
if (typeof fileData === 'number') {
obj.id = fileData;
} else if (typeof fileData === 'string') {
obj.url = fileData;
} else if (isObjectSafe(fileData)) {
obj = fileData;
} else {
throw new Error(`Invalid data format '${typeof fileData}' to import media. Only 'string', 'number', 'object' are accepted.`);
}
let file = await findFile(obj, user, allowedFileTypes);
if (file && !isExtensionAllowed(file.ext.substring(1), allowedFileTypes)) {
file = null;
}
return file;
};
/**
* Find a file.
* @param {Object} filters
* @param {number} [filters.id] - File id.
* @param {string} [filters.hash] - File hash.
* @param {string} [filters.name] - File name.
* @param {string} [filters.url]
* @param {string} [filters.alternativeText]
* @param {string} [filters.caption]
* @param {Object} user
* @returns
*/
const findFile = async ({ id, hash, name, url, alternativeText, caption }, user, allowedFileTypes) => {
let file = null;
if (!file && id) {
file = await strapi.entityService.findOne('plugin::upload.file', id);
}
if (!file && hash) {
[file] = await strapi.entityService.findMany('plugin::upload.file', { filters: { hash }, limit: 1 });
}
if (!file && name) {
[file] = await strapi.entityService.findMany('plugin::upload.file', { filters: { name }, limit: 1 });
}
if (!file && url) {
const checkResult = isValidFileUrl(url, allowedFileTypes);
if (checkResult.isValid) {
file = await findFile({ hash: checkResult.fileData.hash, name: checkResult.fileData.fileName }, user, allowedFileTypes);
if (!file) {
file = await importFile({ id, url: checkResult.fileData.rawUrl, name, alternativeText, caption }, user);
}
}
}
return file;
};
const importFile = async ({ id, url, name, alternativeText, caption }, user) => {
let file;
try {
file = await fetchFile(url);
let [uploadedFile] = await strapi
.plugin('upload')
.service('upload')
.upload(
{
files: {
name: file.name,
type: file.type,
size: file.size,
path: file.path,
},
data: {
fileInfo: {
name: name || file.name,
alternativeText: alternativeText || '',
caption: caption || '',
},
},
},
{ user },
);
if (id) {
uploadedFile = await strapi.db.query('plugin::upload.file').update({
where: { id: uploadedFile.id },
data: { id },
});
}
return uploadedFile;
} catch (err) {
strapi.log.error(err);
throw err;
} finally {
deleteFileIfExists(file?.path);
}
};
const fetchFile = async (url) => {
try {
const response = await fetch(url);
const contentType = response.headers.get('content-type').split(';')[0];
const contentLength = parseInt(response.headers.get('content-length')) || 0;
const buffer = await response.buffer();
const fileData = getFileDataFromRawUrl(url);
const filePath = await writeFile(fileData.name, buffer);
return {
name: fileData.name,
type: contentType,
size: contentLength,
path: filePath,
};
} catch (error) {
throw new Error(`Tried to fetch file from url ${url} but failed with error: ${error.message}`);
}
};
const writeFile = async (name, content) => {
const tmpWorkingDirectory = await fse.mkdtemp(path.join(os.tmpdir(), 'strapi-upload-'));
const filePath = path.join(tmpWorkingDirectory, name);
try {
fs.writeFileSync(filePath, content);
return filePath;
} catch (err) {
strapi.log.error(err);
throw err;
}
};
const deleteFileIfExists = (filePath) => {
if (filePath && fs.existsSync(filePath)) {
fs.rmSync(filePath);
}
};
const isValidFileUrl = (url, allowedFileTypes) => {
try {
const fileData = getFileDataFromRawUrl(url);
return {
isValid: isExtensionAllowed(fileData.extension, allowedFileTypes),
fileData: {
hash: fileData.hash,
fileName: fileData.name,
rawUrl: url,
},
};
} catch (err) {
strapi.log.error(err);
return {
isValid: false,
fileData: {
hash: '',
fileName: '',
rawUrl: '',
},
};
}
};
const isExtensionAllowed = (ext, allowedFileTypes) => {
const checkers = allowedFileTypes.map(getFileTypeChecker);
return checkers.some((checker) => checker(ext));
};
const ALLOWED_AUDIOS = ['mp3', 'wav', 'ogg'];
const ALLOWED_IMAGES = ['png', 'gif', 'jpg', 'jpeg', 'svg', 'bmp', 'tif', 'tiff'];
const ALLOWED_VIDEOS = ['mp4', 'avi'];
/** See Strapi file allowedTypes for object keys. */
const fileTypeCheckers = {
any: (ext) => true,
audios: (ext) => ALLOWED_AUDIOS.includes(ext),
files: (ext) => true,
images: (ext) => ALLOWED_IMAGES.includes(ext),
videos: (ext) => ALLOWED_VIDEOS.includes(ext),
};
const getFileTypeChecker = (type) => {
const checker = fileTypeCheckers[type];
if (!checker) {
throw new Error(`Strapi file type ${type} not handled.`);
}
return checker;
};
const getFileDataFromRawUrl = (rawUrl) => {
const parsedUrl = new URL(decodeURIComponent(rawUrl));
const name = trim(parsedUrl.pathname, '/').replace(/\//g, '-');
const extension = parsedUrl.pathname.split('.').pop().toLowerCase();
const hash = last(parsedUrl.pathname.split('/')).slice(0, -(extension.length + 1));
return {
hash,
name,
extension,
};
};
module.exports = {
findOrImportFile,
};

View File

@ -0,0 +1,9 @@
'use strict';
const exportService = require('./export');
const importService = require('./import');
module.exports = {
export: exportService,
import: importService,
};

View File

@ -0,0 +1,28 @@
const _ = require('lodash');
const errorCodes = {};
const errorMessages = {};
Object.keys(errorMessages).forEach(
(k) =>
(errorMessages[k] = _.template(errorMessages[k], {
interpolate: /\{\s*(\S+)\s*\}/g,
})),
);
class BusinessError extends Error {
constructor(errorCodeOrMessage, interpolations) {
const isErrorCode = !!errorCodes[errorCodeOrMessage];
super(isErrorCode ? errorMessages[errorCodeOrMessage](interpolations) : errorCodeOrMessage);
this.name = this.constructor.name;
this.code = isErrorCode ? errorCodeOrMessage : 'UNDEFINED';
}
}
module.exports = {
BusinessError,
errorCodes,
};

View File

@ -0,0 +1,17 @@
const pluginId = require('./pluginId');
/**
* @typedef {("serverPublicHostname")} ConfigParam
*/
/**
* Get a config parameter.
* @param {ConfigParam} param
*/
const getConfig = (param) => {
return strapi.config.get(`plugin.${pluginId}.${param}`);
};
module.exports = {
getConfig,
};

19
src/server/utils/index.js Normal file
View File

@ -0,0 +1,19 @@
const pluginId = require('./pluginId');
/**
* ServiceName.
* @typedef {("export"|"import")} ServiceName
*/
/**
* Get a plugin service.
* @param {ServiceName} serviceName
* @returns
*/
const getService = (serviceName) => {
return strapi.plugin(pluginId).service(serviceName);
};
module.exports = {
getService,
};

121
src/server/utils/models.js Normal file
View File

@ -0,0 +1,121 @@
'use strict';
const { toArray } = require('../../libs/arrays');
/**
* ModelKind.
* @typedef {("collectionType"|"singleType")} ModelKind
*/
/**
* A model.
* @typedef {Object} Model
* @property {ModelKind} kind
* @property {string} collectionName
* @property {{[k: string]: Attribute}} attributes - Name of the attribute.
*/
/**
* AttributeType.
* @typedef {("boolean"|"component"|"datetime"|"dynamiczone"|"increments"|"media"|"number"|"relation"|"string"|"text")} AttributeType
*/
/**
* AttributeTarget.
* @typedef {("admin::user")} AttributeTarget
*/
/**
* An attribute of a model.
* @typedef {Object} Attribute
* @property {AttributeType} type
* @property {string} name - Name of the attribute.
* @property {string} [target] - Slug of the target model (if type is 'relation').
* @property {string} [component] - Name of the targetted component.
* @property {boolean} [repeatable] - Whether the component is repeatable.
* @property {Array<string>} [components] - Component names of the dynamic zone.
* @property {("audios"|"files"|"images"|"videos")} [allowedTypes] - Allowed file types.
* @property {boolean} [multiple] - Whether there are multiple files.
*/
const getAllSlugs = () => {
return Array.from(strapi.db.metadata)
.map(([collectionName]) => collectionName)
.filter((collectionName) => collectionName.startsWith('api::'));
};
/**
* Get a model.
* @param {string} slug - Slug of the model.
* @returns {Model}
*/
const getModel = (slug) => {
return strapi.getModel(slug);
};
const getModelConfig = (modelOrSlug) => {
const model = getModelFromSlugOrModel(modelOrSlug);
return {
kind: model.kind === 'singleType' ? 'single' : 'collection',
isLocalized: !!model.pluginOptions?.i18n?.localized,
};
};
const getModelFromSlugOrModel = (modelOrSlug) => {
let model = modelOrSlug;
if (typeof model === 'string') {
model = getModel(modelOrSlug);
}
return model;
};
/**
* Get the attributes of a model.
* @param {string} slug - Slug of the model.
* @param {Object} options
* @param {AttributeType | Array<AttributeType>} [options.filterType] - Only attributes matching the type(s) will be kept.
* @param {AttributeType | Array<AttributeType>} [options.filterOutType] - Remove attributes matching the specified type(s).
* @param {AttributeTarget | Array<AttributeTarget>} [options.filterOutTarget] - Remove attributes matching the specified target(s).
* @param {boolean} [options.addIdAttribute] - Add `id` in the returned attributes.
* @returns {Array<Attribute>}
*/
const getModelAttributes = (slug, options = {}) => {
const typesToKeep = options.filterType ? toArray(options.filterType) : [];
const typesToFilterOut = options.filterOutType ? toArray(options.filterOutType) : [];
const filterOutTarget = toArray(options.filterOutTarget || []);
const attributesObj = strapi.getModel(slug).attributes;
let attributes = Object.keys(attributesObj)
.reduce((acc, key) => acc.concat({ ...attributesObj[key], name: key }), [])
.filter((attr) => !typesToFilterOut.includes(attr.type))
.filter((attr) => !filterOutTarget.includes(attr.target));
if (typesToKeep.length) {
attributes = attributes.filter((attr) => typesToKeep.includes(attr.type));
}
if (options.addIdAttribute) {
attributes.unshift({ name: 'id' });
}
return attributes;
};
/**
* Indicate whether an attribute is a dynamic zone.
* @param {Attribute} attribute
* @returns {boolean}
*/
const isAttributeDynamicZone = (attribute) => {
return attribute.components && Array.isArray(attribute.components);
};
module.exports = {
getAllSlugs,
getModel,
getModelConfig,
getModelAttributes,
isAttributeDynamicZone,
};

View File

@ -0,0 +1,5 @@
const pluginPkg = require('../../package.json');
const pluginId = pluginPkg.name.replace(/^(@[^-,.][\w,-]+\/|strapi-)plugin-/i, '');
module.exports = pluginId;

16
tsconfig.json Normal file
View File

@ -0,0 +1,16 @@
{
"compilerOptions": {
"target": "ES6",
"module": "CommonJS",
"strict": true,
"allowJs": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"rootDir": "./src",
"outDir": "./"
},
"$schema": "https://json.schemastore.org/tsconfig",
"include": ["src/**/*"],
"exclude": ["node_modules"]
}

View File

@ -12003,6 +12003,11 @@ typescript@4.6.2:
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4"
integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg==
typescript@^5.0.3:
version "5.0.3"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.0.3.tgz#fe976f0c826a88d0a382007681cbb2da44afdedf"
integrity sha512-xv8mOEDnigb/tN9PSMTwSEqAnUvkoXMQlicOb0IUVDBSQCgBSaAAROUZYy2IcUy5qU6XajK5jjjO7TMWqBTKZA==
uc.micro@^1.0.1, uc.micro@^1.0.5:
version "1.0.6"
resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"