mirror of
https://github.com/Baboo7/strapi-plugin-import-export-entries.git
synced 2025-09-04 00:02:40 -04:00
refactor(import-v2): Map file ids to database ids
This commit is contained in:
Baptiste Studer
parent
e558485bd5
commit
e0b9383c34
@ -50,6 +50,8 @@
|
||||
"@strapi/plugin-seo": "^1.8.0",
|
||||
"@strapi/plugin-users-permissions": "^4.9.0",
|
||||
"@strapi/strapi": "^4.9.0",
|
||||
"@types/fs-extra": "^11.0.1",
|
||||
"@types/node-fetch": "^2.6.3",
|
||||
"eslint": "^8.23.0",
|
||||
"eslint-plugin-jest": "^27.0.1",
|
||||
"eslint-plugin-react": "^7.30.1",
|
||||
|
||||
@ -4,10 +4,12 @@ import omit from 'lodash/omit';
|
||||
import pick from 'lodash/pick';
|
||||
import { toArray } from '../../../libs/arrays';
|
||||
import { ObjectBuilder } from '../../../libs/objects';
|
||||
import { getEntryProp, getModel, getModelAttributes } from '../../utils/models';
|
||||
import { getModel, getModelAttributes } from '../../utils/models';
|
||||
import { SchemaUID } from '@strapi/strapi/lib/types/utils';
|
||||
import { Entry, EntryId } from '../../types';
|
||||
const { findOrImportFile } = require('./utils/file');
|
||||
import { Entry, EntryId, User } from '../../types';
|
||||
import { toPairs } from 'lodash';
|
||||
import { FileEntry, FileId } from './types';
|
||||
import { findOrImportFile } from './utils/file';
|
||||
|
||||
type Import = {
|
||||
version: 2;
|
||||
@ -15,12 +17,9 @@ type Import = {
|
||||
};
|
||||
type ImportData = ImportDataSlugEntries;
|
||||
type ImportDataSlugEntries = {
|
||||
[slug in SchemaUID]: {
|
||||
[key: string]: {
|
||||
[attribute: string]: string | number | string[] | number[] | null;
|
||||
};
|
||||
};
|
||||
[slug in SchemaUID]: SlugEntries;
|
||||
};
|
||||
type SlugEntries = Record<FileId, FileEntry>;
|
||||
|
||||
type ImportFailures = {
|
||||
/** Error raised. */
|
||||
@ -29,7 +28,23 @@ type ImportFailures = {
|
||||
data: any;
|
||||
};
|
||||
|
||||
type User = any;
|
||||
class IdMapper {
|
||||
private mapping: {
|
||||
[slug in SchemaUID]?: Map<string | number, string | number>;
|
||||
} = {};
|
||||
|
||||
public getMapping(slug: SchemaUID, fileId: string | number) {
|
||||
return this.mapping[slug]?.get(`${fileId}`);
|
||||
}
|
||||
|
||||
public setMapping(slug: SchemaUID, fileId: string | number, dbId: string | number) {
|
||||
if (!this.mapping[slug]) {
|
||||
this.mapping[slug] = new Map<string | number, string | number>();
|
||||
}
|
||||
|
||||
this.mapping[slug]!.set(`${fileId}`, dbId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Import data.
|
||||
@ -53,36 +68,42 @@ const importDataV2 = async (
|
||||
|
||||
const slugs: SchemaUID[] = Object.keys(data) as SchemaUID[];
|
||||
let failures: ImportFailures[] = [];
|
||||
const fileIdToDbId = new IdMapper();
|
||||
|
||||
// Import without setting relations.
|
||||
for (const slugFromFile of slugs) {
|
||||
if (slugFromFile === 'plugin::upload.file') {
|
||||
const res = await importMedia(Object.values(data[slugFromFile]) as unknown as Entry[], { user });
|
||||
const res = await importMedia(data[slugFromFile], { user, fileIdToDbId });
|
||||
failures.push(...res.failures);
|
||||
} else {
|
||||
const res = await importOtherSlug(Object.values(data[slugFromFile]) as unknown as Entry[], {
|
||||
const res = await importOtherSlug(data[slugFromFile], {
|
||||
slug: slugFromFile,
|
||||
user,
|
||||
// Keep behavior of `idField` of version 1.
|
||||
...(slugFromFile === slug ? { idField } : {}),
|
||||
importStage: 'simpleAttributes',
|
||||
fileIdToDbId,
|
||||
});
|
||||
failures.push(...res.failures);
|
||||
}
|
||||
}
|
||||
|
||||
// Set relations relations.
|
||||
// TODO: prevent importing media twice
|
||||
// const SLUGS_TO_SKIP: SchemaUID[] = ['plugin::upload.file'];
|
||||
// for (const slugFromFile of slugs.filter(slug => !SLUGS_TO_SKIP.includes(slug))) {
|
||||
for (const slugFromFile of slugs) {
|
||||
if (slugFromFile === 'plugin::upload.file') {
|
||||
// TODO: are media imported twice?
|
||||
const res = await importMedia(Object.values(data[slugFromFile]) as unknown as Entry[], { user });
|
||||
const res = await importMedia(data[slugFromFile], { user, fileIdToDbId });
|
||||
failures.push(...res.failures);
|
||||
} else {
|
||||
const res = await importOtherSlug(Object.values(data[slugFromFile]) as unknown as Entry[], {
|
||||
const res = await importOtherSlug(data[slugFromFile], {
|
||||
slug: slugFromFile,
|
||||
user,
|
||||
// Keep behavior of `idField` of version 1.
|
||||
...(slugFromFile === slug ? { idField } : {}),
|
||||
importStage: 'relationAttributes',
|
||||
fileIdToDbId,
|
||||
});
|
||||
failures.push(...res.failures);
|
||||
}
|
||||
@ -101,15 +122,20 @@ const importDataV2 = async (
|
||||
return { failures };
|
||||
};
|
||||
|
||||
const importMedia = async (fileData: Entry[], { user }: { user: User }): Promise<{ failures: ImportFailures[] }> => {
|
||||
const importMedia = async (slugEntries: SlugEntries, { user, fileIdToDbId }: { user: User; fileIdToDbId: IdMapper }): Promise<{ failures: ImportFailures[] }> => {
|
||||
const failures: ImportFailures[] = [];
|
||||
for (let fileDatum of fileData) {
|
||||
let res;
|
||||
|
||||
const fileEntries = toPairs(slugEntries);
|
||||
|
||||
for (let [fileId, fileEntry] of fileEntries) {
|
||||
try {
|
||||
await findOrImportFile(fileDatum, user, { allowedFileTypes: ['any'] });
|
||||
const dbEntry = await findOrImportFile(fileEntry, user, { allowedFileTypes: ['any'] });
|
||||
if (dbEntry) {
|
||||
fileIdToDbId.setMapping('plugin::upload.file', fileId, dbEntry?.id);
|
||||
}
|
||||
} catch (err: any) {
|
||||
strapi.log.error(err);
|
||||
failures.push({ error: err.message, data: fileDatum });
|
||||
failures.push({ error: err.message, data: fileEntry });
|
||||
}
|
||||
}
|
||||
|
||||
@ -121,19 +147,21 @@ const importMedia = async (fileData: Entry[], { user }: { user: User }): Promise
|
||||
type ImportStage = 'simpleAttributes' | 'relationAttributes';
|
||||
|
||||
const importOtherSlug = async (
|
||||
data: Entry[],
|
||||
{ slug, user, idField, importStage }: { slug: SchemaUID; user: User; idField?: string; importStage: ImportStage },
|
||||
slugEntries: SlugEntries,
|
||||
{ slug, user, idField, importStage, fileIdToDbId }: { slug: SchemaUID; user: User; idField?: string; importStage: ImportStage; fileIdToDbId: IdMapper },
|
||||
): Promise<{ failures: ImportFailures[] }> => {
|
||||
let fileEntries = toPairs(slugEntries);
|
||||
|
||||
// Sort localized data with default locale first.
|
||||
const sortDataByLocale = async () => {
|
||||
const schema = getModel(slug);
|
||||
|
||||
if (schema.pluginOptions?.i18n?.localized) {
|
||||
const defaultLocale = await strapi.plugin('i18n').service('locales').getDefaultLocale();
|
||||
data = data.sort((dataA, dataB) => {
|
||||
if (dataA?.locale === defaultLocale && dataB?.locale === defaultLocale) {
|
||||
fileEntries = fileEntries.sort((dataA, dataB) => {
|
||||
if (dataA[1].locale === defaultLocale && dataB[1].locale === defaultLocale) {
|
||||
return 0;
|
||||
} else if (dataA?.locale === defaultLocale) {
|
||||
} else if (dataA[1].locale === defaultLocale) {
|
||||
return -1;
|
||||
}
|
||||
return 1;
|
||||
@ -143,12 +171,12 @@ const importOtherSlug = async (
|
||||
await sortDataByLocale();
|
||||
|
||||
const failures: ImportFailures[] = [];
|
||||
for (let datum of data) {
|
||||
for (let [fileId, fileEntry] of fileEntries) {
|
||||
try {
|
||||
await updateOrCreate(user, slug, datum, idField, { importStage });
|
||||
await updateOrCreate(user, slug, fileId, fileEntry, idField, { importStage, fileIdToDbId });
|
||||
} catch (err: any) {
|
||||
strapi.log.error(err);
|
||||
failures.push({ error: err.message, data: datum });
|
||||
failures.push({ error: err.message, data: fileEntry });
|
||||
}
|
||||
}
|
||||
|
||||
@ -157,163 +185,198 @@ const importOtherSlug = async (
|
||||
};
|
||||
};
|
||||
|
||||
const updateOrCreate = async (user: User, slug: SchemaUID, datumArg: Entry, idField = 'id', { importStage }: { importStage: ImportStage }) => {
|
||||
const updateOrCreate = async (
|
||||
user: User,
|
||||
slug: SchemaUID,
|
||||
fileId: FileId,
|
||||
fileEntryArg: FileEntry,
|
||||
idField = 'id',
|
||||
{ importStage, fileIdToDbId }: { importStage: ImportStage; fileIdToDbId: IdMapper },
|
||||
) => {
|
||||
const schema = getModel(slug);
|
||||
let datum = cloneDeep(datumArg);
|
||||
let fileEntry = cloneDeep(fileEntryArg);
|
||||
|
||||
if (importStage == 'simpleAttributes') {
|
||||
const attributeNames = getModelAttributes(slug, { filterOutType: ['component', 'dynamiczone', 'media', 'relation'] })
|
||||
.map(({ name }) => name)
|
||||
.concat('id', 'localizations', 'locale');
|
||||
datum = pick(datum, attributeNames) as Entry;
|
||||
fileEntry = pick(fileEntry, attributeNames);
|
||||
} else if (importStage === 'relationAttributes') {
|
||||
const attributeNames = getModelAttributes(slug, { filterType: ['component', 'dynamiczone', 'media', 'relation'] })
|
||||
.map(({ name }) => name)
|
||||
.concat('id', 'localizations', 'locale');
|
||||
datum = pick(datum, attributeNames) as Entry;
|
||||
fileEntry = pick(fileEntry, attributeNames);
|
||||
|
||||
// TODO: update ids using mapping file id => db id, once components PR merged
|
||||
}
|
||||
|
||||
// TODO: handle components create or update?
|
||||
if (schema.modelType === 'contentType' && schema.kind === 'singleType') {
|
||||
await updateOrCreateSingleTypeEntry(user, slug, datum, { importStage });
|
||||
await updateOrCreateSingleTypeEntry(user, slug, fileId, fileEntry, { importStage, fileIdToDbId });
|
||||
} else {
|
||||
await updateOrCreateEntry(user, slug, datum, { idField, importStage });
|
||||
await updateOrCreateEntry(user, slug, fileId, fileEntry, { idField, importStage, fileIdToDbId });
|
||||
}
|
||||
};
|
||||
|
||||
const updateOrCreateEntry = async (user: User, slug: SchemaUID, datum: Entry, { idField, importStage }: { idField: string; importStage: ImportStage }) => {
|
||||
const updateOrCreateEntry = async (
|
||||
user: User,
|
||||
slug: SchemaUID,
|
||||
fileId: FileId,
|
||||
fileEntry: FileEntry,
|
||||
{ idField, importStage, fileIdToDbId }: { idField: string; importStage: ImportStage; fileIdToDbId: IdMapper },
|
||||
) => {
|
||||
const schema = getModel(slug);
|
||||
|
||||
const whereBuilder = new ObjectBuilder();
|
||||
if (getEntryProp(datum, idField)) {
|
||||
whereBuilder.extend({ [idField]: getEntryProp(datum, idField) });
|
||||
if (fileIdToDbId.getMapping(slug, fileId)) {
|
||||
whereBuilder.extend({ id: fileIdToDbId.getMapping(slug, fileId) });
|
||||
} else if (fileEntry[idField]) {
|
||||
whereBuilder.extend({ [idField]: fileEntry[idField] });
|
||||
}
|
||||
const where = whereBuilder.get();
|
||||
|
||||
if (!schema.pluginOptions?.i18n?.localized) {
|
||||
let entry = await strapi.db.query(slug).findOne({ where });
|
||||
let dbEntry: Entry = await strapi.db.query(slug).findOne({ where });
|
||||
|
||||
if (!entry) {
|
||||
await strapi.entityService.create(slug, { data: datum });
|
||||
if (!dbEntry) {
|
||||
dbEntry = await strapi.entityService.create(slug, { data: fileEntry });
|
||||
} else {
|
||||
await updateEntry(slug, entry.id, datum, { importStage });
|
||||
dbEntry = await updateEntry(slug, dbEntry.id, fileEntry, { importStage });
|
||||
}
|
||||
fileIdToDbId.setMapping(slug, fileId, dbEntry.id);
|
||||
} else {
|
||||
if (!datum.locale) {
|
||||
throw new Error(`No locale set to import entry for slug ${slug} (data ${JSON.stringify(datum)})`);
|
||||
if (!fileEntry.locale) {
|
||||
throw new Error(`No locale set to import entry for slug ${slug} (data ${JSON.stringify(fileEntry)})`);
|
||||
}
|
||||
|
||||
const defaultLocale = await strapi.plugin('i18n').service('locales').getDefaultLocale();
|
||||
const isDatumInDefaultLocale = datum.locale === defaultLocale;
|
||||
const isDatumInDefaultLocale = fileEntry.locale === defaultLocale;
|
||||
|
||||
let entryDefaultLocale = null;
|
||||
let entry = await strapi.db.query(slug).findOne({ where, populate: ['localizations'] });
|
||||
let dbEntryDefaultLocaleId: EntryId | null = null;
|
||||
let dbEntry: Entry | null = await strapi.db.query(slug).findOne({ where, populate: ['localizations'] });
|
||||
if (isDatumInDefaultLocale) {
|
||||
entryDefaultLocale = entry;
|
||||
dbEntryDefaultLocaleId = dbEntry?.id || null;
|
||||
} else {
|
||||
if (entry) {
|
||||
// If `entry` has been found, `entry` holds the data for the default locale and
|
||||
if (dbEntry) {
|
||||
// If `dbEntry` has been found, `dbEntry` holds the data for the default locale and
|
||||
// the data for other locales in its `localizations` attribute.
|
||||
const localizedEntries = [entry, ...(entry?.localizations || [])];
|
||||
entryDefaultLocale = localizedEntries.find((e) => e.locale === defaultLocale);
|
||||
entry = localizedEntries.find((e) => e.locale === datum.locale);
|
||||
const localizedEntries = [dbEntry, ...(dbEntry?.localizations || [])];
|
||||
dbEntryDefaultLocaleId = localizedEntries.find((e: Entry) => e.locale === defaultLocale)?.id || null;
|
||||
dbEntry = localizedEntries.find((e: Entry) => e.locale === fileEntry.locale) || null;
|
||||
} else {
|
||||
// Otherwise try to find entry for default locale through localized siblings.
|
||||
let localizationIdx = 0;
|
||||
const localizations = datum?.localizations || [];
|
||||
while (localizationIdx < localizations.length && !entryDefaultLocale && !entry) {
|
||||
const id = localizations[localizationIdx];
|
||||
const localizedEntry = await strapi.db.query(slug).findOne({ where: { id }, populate: ['localizations'] });
|
||||
// Otherwise try to find dbEntry for default locale through localized siblings.
|
||||
let idx = 0;
|
||||
const fileLocalizationsIds = (fileEntry?.localizations as EntryId[]) || [];
|
||||
while (idx < fileLocalizationsIds.length && !dbEntryDefaultLocaleId && !dbEntry) {
|
||||
const dbId = fileIdToDbId.getMapping(slug, fileLocalizationsIds[idx]);
|
||||
const localizedEntry: Entry = await strapi.db.query(slug).findOne({ where: { id: dbId }, populate: ['localizations'] });
|
||||
const localizedEntries = localizedEntry != null ? [localizedEntry, ...(localizedEntry?.localizations || [])] : [];
|
||||
if (!entryDefaultLocale) {
|
||||
entryDefaultLocale = localizedEntries.find((e) => e.locale === defaultLocale);
|
||||
if (!dbEntryDefaultLocaleId) {
|
||||
dbEntryDefaultLocaleId = localizedEntries.find((e: Entry) => e.locale === defaultLocale)?.id || null;
|
||||
}
|
||||
if (!entry) {
|
||||
entry = localizedEntries.find((e) => e.locale === datum.locale);
|
||||
if (!dbEntry) {
|
||||
dbEntry = localizedEntries.find((e: Entry) => e.locale === fileEntry.locale) || null;
|
||||
}
|
||||
localizationIdx += 1;
|
||||
idx += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
datum = omit(datum, ['localizations']);
|
||||
if (isEmpty(omit(datum, ['id']))) {
|
||||
fileEntry = omit(fileEntry, ['localizations']);
|
||||
if (isEmpty(omit(fileEntry, ['id']))) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (isDatumInDefaultLocale) {
|
||||
if (!entryDefaultLocale) {
|
||||
await strapi.entityService.create(slug, { data: datum });
|
||||
if (!dbEntryDefaultLocaleId) {
|
||||
const createdEntry = await strapi.entityService.create(slug, { data: fileEntry });
|
||||
fileIdToDbId.setMapping(slug, fileId, createdEntry.id);
|
||||
} else {
|
||||
await strapi.entityService.update(slug, entryDefaultLocale.id, { data: omit({ ...datum }, ['id']) });
|
||||
const updatedEntry = await strapi.entityService.update(slug, dbEntryDefaultLocaleId, { data: omit({ ...fileEntry }, ['id']) });
|
||||
fileIdToDbId.setMapping(slug, fileId, updatedEntry.id);
|
||||
}
|
||||
} else {
|
||||
if (!entryDefaultLocale) {
|
||||
throw new Error(`Could not find default locale entry to import localization for slug ${slug} (data ${JSON.stringify(datum)})`);
|
||||
if (!dbEntryDefaultLocaleId) {
|
||||
throw new Error(`Could not find default locale entry to import localization for slug ${slug} (data ${JSON.stringify(fileEntry)})`);
|
||||
}
|
||||
|
||||
if (!entry) {
|
||||
if (!dbEntry) {
|
||||
const insertLocalizedEntry = strapi.plugin('i18n').service('core-api').createCreateLocalizationHandler(getModel(slug));
|
||||
await insertLocalizedEntry({ id: entryDefaultLocale.id, data: omit({ ...datum }, ['id']) });
|
||||
const createdEntry: Entry = await insertLocalizedEntry({ id: dbEntryDefaultLocaleId, data: omit({ ...fileEntry }, ['id']) });
|
||||
fileIdToDbId.setMapping(slug, fileId, createdEntry.id);
|
||||
} else {
|
||||
await strapi.entityService.update(slug, entry.id, { data: omit({ ...datum }, ['id']) });
|
||||
const updatedEntry = await strapi.entityService.update(slug, dbEntry.id, { data: omit({ ...fileEntry }, ['id']) });
|
||||
fileIdToDbId.setMapping(slug, fileId, updatedEntry.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const updateOrCreateSingleTypeEntry = async (user: User, slug: SchemaUID, datum: Entry, { importStage }: { importStage: ImportStage }) => {
|
||||
const updateOrCreateSingleTypeEntry = async (
|
||||
user: User,
|
||||
slug: SchemaUID,
|
||||
fileId: FileId,
|
||||
fileEntry: FileEntry,
|
||||
{ importStage, fileIdToDbId }: { importStage: ImportStage; fileIdToDbId: IdMapper },
|
||||
) => {
|
||||
const schema = getModel(slug);
|
||||
|
||||
if (!schema.pluginOptions?.i18n?.localized) {
|
||||
let entry: Entry = await strapi.db
|
||||
let dbEntry: Entry = await strapi.db
|
||||
.query(slug)
|
||||
.findMany({})
|
||||
.then((entries) => toArray(entries)?.[0]);
|
||||
|
||||
if (!entry) {
|
||||
await strapi.entityService.create(slug, { data: datum });
|
||||
if (!dbEntry) {
|
||||
const createdEntry = await strapi.entityService.create(slug, { data: fileEntry });
|
||||
fileIdToDbId.setMapping(slug, fileId, createdEntry.id);
|
||||
} else {
|
||||
await updateEntry(slug, entry.id, datum, { importStage });
|
||||
const updatedEntry = await updateEntry(slug, dbEntry.id, fileEntry, { importStage });
|
||||
fileIdToDbId.setMapping(slug, fileId, updatedEntry.id);
|
||||
}
|
||||
} else {
|
||||
const defaultLocale = await strapi.plugin('i18n').service('locales').getDefaultLocale();
|
||||
const isDatumInDefaultLocale = !datum.locale || datum.locale === defaultLocale;
|
||||
const isDatumInDefaultLocale = !fileEntry.locale || fileEntry.locale === defaultLocale;
|
||||
|
||||
datum = omit(datum, ['localizations']);
|
||||
if (isEmpty(omit(datum, ['id']))) {
|
||||
fileEntry = omit(fileEntry, ['localizations']);
|
||||
if (isEmpty(omit(fileEntry, ['id']))) {
|
||||
return;
|
||||
}
|
||||
|
||||
let entryDefaultLocale = await strapi.db.query(slug).findOne({ where: { locale: defaultLocale } });
|
||||
if (!entryDefaultLocale) {
|
||||
entryDefaultLocale = await strapi.entityService.create(slug, { data: { ...datum, locale: defaultLocale } });
|
||||
entryDefaultLocale = await strapi.entityService.create(slug, { data: { ...fileEntry, locale: defaultLocale } });
|
||||
}
|
||||
|
||||
if (isDatumInDefaultLocale) {
|
||||
if (!entryDefaultLocale) {
|
||||
await strapi.entityService.create(slug, { data: datum });
|
||||
const createdEntry: Entry = await strapi.entityService.create(slug, { data: fileEntry });
|
||||
fileIdToDbId.setMapping(slug, fileId, createdEntry.id);
|
||||
} else {
|
||||
await strapi.entityService.update(slug, entryDefaultLocale.id, { data: datum });
|
||||
const updatedEntry: Entry = await strapi.entityService.update(slug, entryDefaultLocale.id, { data: fileEntry });
|
||||
fileIdToDbId.setMapping(slug, fileId, updatedEntry.id);
|
||||
}
|
||||
} else {
|
||||
const entryLocale = await strapi.db.query(slug).findOne({ where: { locale: datum.locale } });
|
||||
let datumLocale = { ...entryLocale, ...datum };
|
||||
const entryLocale = await strapi.db.query(slug).findOne({ where: { locale: fileEntry.locale } });
|
||||
let datumLocale = { ...entryLocale, ...fileEntry };
|
||||
|
||||
await strapi.db.query(slug).delete({ where: { locale: datum.locale } });
|
||||
await strapi.db.query(slug).delete({ where: { locale: fileEntry.locale } });
|
||||
|
||||
const insertLocalizedEntry = strapi.plugin('i18n').service('core-api').createCreateLocalizationHandler(getModel(slug));
|
||||
await insertLocalizedEntry({ id: entryDefaultLocale.id, data: datumLocale });
|
||||
const createdEntry: Entry = await insertLocalizedEntry({ id: entryDefaultLocale.id, data: datumLocale });
|
||||
fileIdToDbId.setMapping(slug, fileId, createdEntry.id);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const updateEntry = async (slug: SchemaUID, id: EntryId, datum: Entry, { importStage }: { importStage: ImportStage }) => {
|
||||
const updateEntry = async (slug: SchemaUID, dbId: EntryId, fileEntry: FileEntry, { importStage }: { importStage: ImportStage }): Promise<Entry> => {
|
||||
if (importStage === 'simpleAttributes') {
|
||||
await strapi.entityService.update(slug, id, { data: omit(datum, ['id']) });
|
||||
// Use entity service to validate values of attributes
|
||||
return strapi.entityService.update(slug, dbId, { data: omit(fileEntry, ['id']) });
|
||||
} else if (importStage === 'relationAttributes') {
|
||||
await strapi.db.query(slug).update({ where: { id }, data: omit(datum, ['id']) });
|
||||
return strapi.db.query(slug).update({ where: { id: dbId }, data: omit(fileEntry, ['id']) });
|
||||
}
|
||||
|
||||
throw new Error(`Unhandled importStage ${importStage}`);
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
|
||||
17
src/server/services/import/types/index.ts
Normal file
17
src/server/services/import/types/index.ts
Normal file
@ -0,0 +1,17 @@
|
||||
import { SchemaUID } from '@strapi/strapi/lib/types/utils';
|
||||
|
||||
export { FileEntry, FileId };
|
||||
|
||||
type FileId = string;
|
||||
type FileEntry = {
|
||||
[attribute: string]:
|
||||
| string
|
||||
| number
|
||||
| string[]
|
||||
| number[]
|
||||
| {
|
||||
__component: SchemaUID;
|
||||
id: FileId;
|
||||
}
|
||||
| null;
|
||||
};
|
||||
@ -1,55 +1,56 @@
|
||||
const fs = require('fs');
|
||||
const fse = require('fs-extra');
|
||||
const last = require('lodash/last');
|
||||
const trim = require('lodash/trim');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const fetch = require('node-fetch');
|
||||
import { Entry, MediaEntry, User } from '../../../types';
|
||||
import { FileEntry } from '../types';
|
||||
import fs from 'fs';
|
||||
import fse from 'fs-extra';
|
||||
import last from 'lodash/last';
|
||||
import trim from 'lodash/trim';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import fetch from 'node-fetch';
|
||||
import { isObjectSafe } from '../../../../libs/objects';
|
||||
|
||||
const { isObjectSafe } = require('../../../../libs/objects');
|
||||
export { findOrImportFile };
|
||||
|
||||
/**
|
||||
* Find or import a file.
|
||||
* @param {*} fileData - Strapi file data.
|
||||
* @param {*} user - Strapi user.
|
||||
* @param {Object} options
|
||||
* @param {Array<string>} options.allowedFileTypes - File types the file should match (see Strapi file allowedTypes).
|
||||
* @returns
|
||||
*/
|
||||
const findOrImportFile = async (fileData, user, { allowedFileTypes }) => {
|
||||
let obj = {};
|
||||
if (typeof fileData === 'number') {
|
||||
obj.id = fileData;
|
||||
} else if (typeof fileData === 'string') {
|
||||
obj.url = fileData;
|
||||
} else if (isObjectSafe(fileData)) {
|
||||
obj = fileData;
|
||||
module.exports = {
|
||||
findOrImportFile,
|
||||
};
|
||||
|
||||
type AllowedMediaTypes = keyof typeof fileTypeCheckers;
|
||||
type FileEntryMedia = {
|
||||
id: string;
|
||||
hash: string;
|
||||
name: string;
|
||||
url: string;
|
||||
alternativeText: string;
|
||||
caption: string;
|
||||
};
|
||||
|
||||
async function findOrImportFile(fileEntry: FileEntry, user: User, { allowedFileTypes }: { allowedFileTypes: AllowedMediaTypes[] }): Promise<Entry | null> {
|
||||
let obj: Partial<FileEntryMedia> = {};
|
||||
if (typeof fileEntry === 'number') {
|
||||
obj.id = fileEntry;
|
||||
} else if (typeof fileEntry === 'string') {
|
||||
obj.url = fileEntry;
|
||||
} else if (isObjectSafe(fileEntry)) {
|
||||
obj = fileEntry;
|
||||
} else {
|
||||
throw new Error(`Invalid data format '${typeof fileData}' to import media. Only 'string', 'number', 'object' are accepted.`);
|
||||
throw new Error(`Invalid data format '${typeof fileEntry}' to import media. Only 'string', 'number', 'object' are accepted.`);
|
||||
}
|
||||
|
||||
let file = await findFile(obj, user, allowedFileTypes);
|
||||
let file: MediaEntry | null = await findFile(obj, user, allowedFileTypes);
|
||||
|
||||
if (file && !isExtensionAllowed(file.ext.substring(1), allowedFileTypes)) {
|
||||
file = null;
|
||||
}
|
||||
|
||||
return file;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a file.
|
||||
* @param {Object} filters
|
||||
* @param {number} [filters.id] - File id.
|
||||
* @param {string} [filters.hash] - File hash.
|
||||
* @param {string} [filters.name] - File name.
|
||||
* @param {string} [filters.url]
|
||||
* @param {string} [filters.alternativeText]
|
||||
* @param {string} [filters.caption]
|
||||
* @param {Object} user
|
||||
* @returns
|
||||
*/
|
||||
const findFile = async ({ id, hash, name, url, alternativeText, caption }, user, allowedFileTypes) => {
|
||||
const findFile = async (
|
||||
{ id, hash, name, url, alternativeText, caption }: Partial<FileEntryMedia>,
|
||||
user: User,
|
||||
allowedFileTypes: AllowedMediaTypes[],
|
||||
): Promise<MediaEntry | null> => {
|
||||
let file = null;
|
||||
|
||||
if (!file && id) {
|
||||
@ -67,7 +68,7 @@ const findFile = async ({ id, hash, name, url, alternativeText, caption }, user,
|
||||
file = await findFile({ hash: checkResult.fileData.hash, name: checkResult.fileData.fileName }, user, allowedFileTypes);
|
||||
|
||||
if (!file) {
|
||||
file = await importFile({ id, url: checkResult.fileData.rawUrl, name, alternativeText, caption }, user);
|
||||
file = await importFile({ id: id!, url: checkResult.fileData.rawUrl, name: name!, alternativeText: alternativeText!, caption: caption! }, user);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -75,7 +76,10 @@ const findFile = async ({ id, hash, name, url, alternativeText, caption }, user,
|
||||
return file;
|
||||
};
|
||||
|
||||
const importFile = async ({ id, url, name, alternativeText, caption }, user) => {
|
||||
const importFile = async (
|
||||
{ id, url, name, alternativeText, caption }: { id: string; url: string; name: string; alternativeText: string; caption: string },
|
||||
user: User,
|
||||
): Promise<MediaEntry> => {
|
||||
let file;
|
||||
try {
|
||||
file = await fetchFile(url);
|
||||
@ -114,15 +118,24 @@ const importFile = async ({ id, url, name, alternativeText, caption }, user) =>
|
||||
strapi.log.error(err);
|
||||
throw err;
|
||||
} finally {
|
||||
deleteFileIfExists(file?.path);
|
||||
if (file?.path) {
|
||||
deleteFileIfExists(file?.path);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const fetchFile = async (url) => {
|
||||
const fetchFile = async (
|
||||
url: string,
|
||||
): Promise<{
|
||||
name: string;
|
||||
type: string;
|
||||
size: number;
|
||||
path: string;
|
||||
}> => {
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
const contentType = response.headers.get('content-type').split(';')[0];
|
||||
const contentLength = parseInt(response.headers.get('content-length')) || 0;
|
||||
const contentType = response.headers.get('content-type')?.split(';')?.[0] || '';
|
||||
const contentLength = parseInt(response.headers.get('content-length') || '0', 10) || 0;
|
||||
const buffer = await response.buffer();
|
||||
const fileData = getFileDataFromRawUrl(url);
|
||||
const filePath = await writeFile(fileData.name, buffer);
|
||||
@ -132,12 +145,12 @@ const fetchFile = async (url) => {
|
||||
size: contentLength,
|
||||
path: filePath,
|
||||
};
|
||||
} catch (error) {
|
||||
} catch (error: any) {
|
||||
throw new Error(`Tried to fetch file from url ${url} but failed with error: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
const writeFile = async (name, content) => {
|
||||
const writeFile = async (name: string, content: Buffer): Promise<string> => {
|
||||
const tmpWorkingDirectory = await fse.mkdtemp(path.join(os.tmpdir(), 'strapi-upload-'));
|
||||
|
||||
const filePath = path.join(tmpWorkingDirectory, name);
|
||||
@ -150,13 +163,23 @@ const writeFile = async (name, content) => {
|
||||
}
|
||||
};
|
||||
|
||||
const deleteFileIfExists = (filePath) => {
|
||||
const deleteFileIfExists = (filePath: string): void => {
|
||||
if (filePath && fs.existsSync(filePath)) {
|
||||
fs.rmSync(filePath);
|
||||
}
|
||||
};
|
||||
|
||||
const isValidFileUrl = (url, allowedFileTypes) => {
|
||||
const isValidFileUrl = (
|
||||
url: string,
|
||||
allowedFileTypes: AllowedMediaTypes[],
|
||||
): {
|
||||
isValid: boolean;
|
||||
fileData: {
|
||||
hash: string;
|
||||
fileName: string;
|
||||
rawUrl: string;
|
||||
};
|
||||
} => {
|
||||
try {
|
||||
const fileData = getFileDataFromRawUrl(url);
|
||||
|
||||
@ -181,7 +204,7 @@ const isValidFileUrl = (url, allowedFileTypes) => {
|
||||
}
|
||||
};
|
||||
|
||||
const isExtensionAllowed = (ext, allowedFileTypes) => {
|
||||
const isExtensionAllowed = (ext: string, allowedFileTypes: AllowedMediaTypes[]) => {
|
||||
const checkers = allowedFileTypes.map(getFileTypeChecker);
|
||||
return checkers.some((checker) => checker(ext));
|
||||
};
|
||||
@ -192,14 +215,14 @@ const ALLOWED_VIDEOS = ['mp4', 'avi'];
|
||||
|
||||
/** See Strapi file allowedTypes for object keys. */
|
||||
const fileTypeCheckers = {
|
||||
any: (ext) => true,
|
||||
audios: (ext) => ALLOWED_AUDIOS.includes(ext),
|
||||
files: (ext) => true,
|
||||
images: (ext) => ALLOWED_IMAGES.includes(ext),
|
||||
videos: (ext) => ALLOWED_VIDEOS.includes(ext),
|
||||
};
|
||||
any: (ext: string) => true,
|
||||
audios: (ext: string) => ALLOWED_AUDIOS.includes(ext),
|
||||
files: (ext: string) => true,
|
||||
images: (ext: string) => ALLOWED_IMAGES.includes(ext),
|
||||
videos: (ext: string) => ALLOWED_VIDEOS.includes(ext),
|
||||
} as const;
|
||||
|
||||
const getFileTypeChecker = (type) => {
|
||||
const getFileTypeChecker = (type: AllowedMediaTypes) => {
|
||||
const checker = fileTypeCheckers[type];
|
||||
if (!checker) {
|
||||
throw new Error(`Strapi file type ${type} not handled.`);
|
||||
@ -207,12 +230,18 @@ const getFileTypeChecker = (type) => {
|
||||
return checker;
|
||||
};
|
||||
|
||||
const getFileDataFromRawUrl = (rawUrl) => {
|
||||
const getFileDataFromRawUrl = (
|
||||
rawUrl: string,
|
||||
): {
|
||||
hash: string;
|
||||
name: string;
|
||||
extension: string;
|
||||
} => {
|
||||
const parsedUrl = new URL(decodeURIComponent(rawUrl));
|
||||
|
||||
const name = trim(parsedUrl.pathname, '/').replace(/\//g, '-');
|
||||
const extension = parsedUrl.pathname.split('.').pop().toLowerCase();
|
||||
const hash = last(parsedUrl.pathname.split('/')).slice(0, -(extension.length + 1));
|
||||
const extension = parsedUrl.pathname.split('.').pop()?.toLowerCase() || '';
|
||||
const hash = last(parsedUrl.pathname.split('/'))?.slice(0, -(extension!.length + 1)) || '';
|
||||
|
||||
return {
|
||||
hash,
|
||||
@ -220,7 +249,3 @@ const getFileDataFromRawUrl = (rawUrl) => {
|
||||
extension,
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
findOrImportFile,
|
||||
};
|
||||
@ -7,6 +7,7 @@ import {
|
||||
DynamicZoneAttribute as StrapiDynamicZoneAttribute,
|
||||
DynamicZoneValue as StrapiDynamicZoneValue,
|
||||
MediaAttribute as StrapiMediaAttribute,
|
||||
MediaValue as StrapiMediaValue,
|
||||
RelationAttribute as StrapiRelationAttribute,
|
||||
RelationValue as StrapiRelationValue,
|
||||
} from '@strapi/strapi';
|
||||
@ -22,13 +23,17 @@ export type {
|
||||
EntryId,
|
||||
ComponentEntry,
|
||||
DynamicZoneEntry,
|
||||
MediaEntry,
|
||||
RelationEntry,
|
||||
Schema,
|
||||
CollectionTypeSchema,
|
||||
SingleTypeSchema,
|
||||
ComponentSchema,
|
||||
User,
|
||||
};
|
||||
|
||||
type User = any;
|
||||
|
||||
type BaseAttribute = { name: string };
|
||||
type Attribute = ComponentAttribute | DynamicZoneAttribute | MediaAttribute | RelationAttribute;
|
||||
type ComponentAttribute = BaseAttribute & (StrapiComponentAttribute<'own-component', true> | StrapiComponentAttribute<'own-component', false>);
|
||||
@ -51,6 +56,7 @@ type RelationAttribute = BaseAttribute &
|
||||
type Entry = ComponentEntry | DynamicZoneEntry | RelationEntry;
|
||||
type ComponentEntry = (WithI18n<StrapiComponentValue<'own-component', true>> & EntryBase) | (WithI18n<StrapiComponentValue<'own-component', false>> & EntryBase);
|
||||
type DynamicZoneEntry = WithI18n<UnwrapArray<StrapiDynamicZoneValue<['own-component']>>> & EntryBase;
|
||||
type MediaEntry = StrapiMediaValue;
|
||||
type RelationEntry =
|
||||
| (WithI18n<StrapiRelationValue<'oneToOne', 'own-collection-type' | 'own-single-type'>> & EntryBase)
|
||||
| (WithI18n<StrapiRelationValue<'oneToMany', 'own-collection-type' | 'own-single-type'>> & EntryBase)
|
||||
|
||||
32
yarn.lock
32
yarn.lock
@ -2698,6 +2698,14 @@
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/fs-extra@^11.0.1":
|
||||
version "11.0.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.1.tgz#f542ec47810532a8a252127e6e105f487e0a6ea5"
|
||||
integrity sha512-MxObHvNl4A69ofaTRU8DFqvgzzv8s9yRtaPPm5gud9HDNvpB3GPQFvNuTWAI59B9huVGV5jXYJwbCsmBsOGYWA==
|
||||
dependencies:
|
||||
"@types/jsonfile" "*"
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/glob@^7.1.1":
|
||||
version "7.2.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb"
|
||||
@ -2777,6 +2785,13 @@
|
||||
resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3"
|
||||
integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==
|
||||
|
||||
"@types/jsonfile@*":
|
||||
version "6.1.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/jsonfile/-/jsonfile-6.1.1.tgz#ac84e9aefa74a2425a0fb3012bdea44f58970f1b"
|
||||
integrity sha512-GSgiRCVeapDN+3pqA35IkQwasaCh/0YFH5dEF6S88iDvEn901DjOeH3/QPY+XYP1DFzDZPvIvfeEgk+7br5png==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/keyv@^3.1.4":
|
||||
version "3.1.4"
|
||||
resolved "https://registry.yarnpkg.com/@types/keyv/-/keyv-3.1.4.tgz#3ccdb1c6751b0c7e52300bcdacd5bcbf8faa75b6"
|
||||
@ -2813,6 +2828,14 @@
|
||||
resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c"
|
||||
integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==
|
||||
|
||||
"@types/node-fetch@^2.6.3":
|
||||
version "2.6.3"
|
||||
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.3.tgz#175d977f5e24d93ad0f57602693c435c57ad7e80"
|
||||
integrity sha512-ETTL1mOEdq/sxUtgtOhKjyB2Irra4cjxksvcMUR5Zr4n+PxVhsCD9WS46oPbHL3et9Zde7CNRr+WUNlcHvsX+w==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
form-data "^3.0.0"
|
||||
|
||||
"@types/node@*":
|
||||
version "18.15.11"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.11.tgz#b3b790f09cb1696cffcec605de025b088fa4225f"
|
||||
@ -6087,6 +6110,15 @@ fork-ts-checker-webpack-plugin@7.2.1:
|
||||
semver "^7.3.5"
|
||||
tapable "^2.2.1"
|
||||
|
||||
form-data@^3.0.0:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f"
|
||||
integrity sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==
|
||||
dependencies:
|
||||
asynckit "^0.4.0"
|
||||
combined-stream "^1.0.8"
|
||||
mime-types "^2.1.12"
|
||||
|
||||
form-data@^4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user