feat(Set Node): Overhaul (#6348)

Github issue / Community forum post (link here to close automatically):
https://github.com/n8n-io/n8n/pull/6348

---------

Co-authored-by: Giulio Andreini <g.andreini@gmail.com>
Co-authored-by: Marcus <marcus@n8n.io>
This commit is contained in:
Michael Kret
2023-09-19 13:16:35 +03:00
committed by GitHub
parent 050ba706d3
commit 3a474552b2
42 changed files with 2626 additions and 410 deletions

View File

@@ -0,0 +1,262 @@
/* eslint-disable n8n-nodes-base/node-filename-against-convention */
import type {
IDataObject,
IExecuteFunctions,
INodeExecutionData,
INodeType,
INodeTypeBaseDescription,
INodeTypeDescription,
} from 'n8n-workflow';
import type { IncludeMods, SetField, SetNodeOptions } from './helpers/interfaces';
import { INCLUDE } from './helpers/interfaces';
import * as raw from './raw.mode';
import * as manual from './manual.mode';
type Mode = 'manual' | 'raw';
const versionDescription: INodeTypeDescription = {
displayName: 'Edit Fields (Set)',
name: 'set',
icon: 'fa:pen',
group: ['input'],
version: 3,
description: 'Change the structure of your items',
subtitle: '={{$parameter["mode"]}}',
defaults: {
name: 'Edit Fields',
color: '#0000FF',
},
inputs: ['main'],
outputs: ['main'],
properties: [
{
displayName: 'Mode',
name: 'mode',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Manual Mapping',
value: 'manual',
description: 'Edit item fields one by one',
action: 'Edit item fields one by one',
},
{
name: 'JSON Output',
value: 'raw',
description: 'Customize item output with JSON',
action: 'Customize item output with JSON',
},
],
default: 'manual',
},
{
displayName: 'Duplicate Item',
name: 'duplicateItem',
type: 'boolean',
default: false,
isNodeSetting: true,
},
{
displayName: 'Duplicate Item Count',
name: 'duplicateCount',
type: 'number',
default: 0,
typeOptions: {
minValue: 0,
},
description:
'How many times the item should be duplicated, mainly used for testing and debugging',
isNodeSetting: true,
displayOptions: {
show: {
duplicateItem: [true],
},
},
},
{
displayName:
'Item duplication is set in the node settings. This option will be ignored when the workflow runs automatically.',
name: 'duplicateWarning',
type: 'notice',
default: '',
displayOptions: {
show: {
duplicateItem: [true],
},
},
},
...raw.description,
...manual.description,
{
displayName: 'Include in Output',
name: 'include',
type: 'options',
description: 'How to select the fields you want to include in your output items',
default: 'all',
options: [
{
name: 'All Input Fields',
value: INCLUDE.ALL,
description: 'Also include all unchanged fields from the input',
},
{
name: 'No Input Fields',
value: INCLUDE.NONE,
description: 'Include only the fields specified above',
},
{
name: 'Selected Input Fields',
value: INCLUDE.SELECTED,
description: 'Also include the fields listed in the parameter “Fields to Include”',
},
{
name: 'All Input Fields Except',
value: INCLUDE.EXCEPT,
description: 'Exclude the fields listed in the parameter “Fields to Exclude”',
},
],
},
{
displayName: 'Fields to Include',
name: 'includeFields',
type: 'string',
default: '',
placeholder: 'e.g. fieldToInclude1,fieldToInclude2',
description:
'Comma-separated list of the field names you want to include in the output. You can drag the selected fields from the input panel.',
requiresDataPath: 'multiple',
displayOptions: {
show: {
include: ['selected'],
},
},
},
{
displayName: 'Fields to Exclude',
name: 'excludeFields',
type: 'string',
default: '',
placeholder: 'e.g. fieldToExclude1,fieldToExclude2',
description:
'Comma-separated list of the field names you want to exclude from the output. You can drag the selected fields from the input panel.',
requiresDataPath: 'multiple',
displayOptions: {
show: {
include: ['except'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'Include Binary Data',
name: 'includeBinary',
type: 'boolean',
default: true,
description: 'Whether binary data should be included if present in the input item',
},
{
displayName: 'Ignore Type Conversion Errors',
name: 'ignoreConversionErrors',
type: 'boolean',
default: false,
description:
'Whether to ignore field type errors and apply a less strict type conversion',
displayOptions: {
show: {
'/mode': ['manual'],
},
},
},
{
displayName: 'Support Dot Notation',
name: 'dotNotation',
type: 'boolean',
default: true,
// eslint-disable-next-line n8n-nodes-base/node-param-description-boolean-without-whether
description:
'By default, dot-notation is used in property names. This means that "a.b" will set the property "b" underneath "a" so { "a": { "b": value} }. If that is not intended this can be deactivated, it will then set { "a.b": value } instead.',
},
],
},
],
};
export class SetV2 implements INodeType {
description: INodeTypeDescription;
constructor(baseDescription: INodeTypeBaseDescription) {
this.description = {
...baseDescription,
...versionDescription,
};
}
async execute(this: IExecuteFunctions) {
const items = this.getInputData();
const mode = this.getNodeParameter('mode', 0) as Mode;
const duplicateItem = this.getNodeParameter('duplicateItem', 0, false) as boolean;
const setNode = { raw, manual };
const returnData: INodeExecutionData[] = [];
const rawData: IDataObject = {};
if (mode === 'raw') {
const jsonOutput = this.getNodeParameter('jsonOutput', 0, '', {
rawExpressions: true,
}) as string | undefined;
if (jsonOutput?.startsWith('=')) {
rawData.jsonOutput = jsonOutput.replace(/^=+/, '');
}
} else {
const workflowFieldsJson = this.getNodeParameter('fields.values', 0, [], {
rawExpressions: true,
}) as SetField[];
for (const entry of workflowFieldsJson) {
if (entry.type === 'objectValue' && (entry.objectValue as string).startsWith('=')) {
rawData[entry.name] = (entry.objectValue as string).replace(/^=+/, '');
}
}
}
for (let i = 0; i < items.length; i++) {
const include = this.getNodeParameter('include', i) as IncludeMods;
const options = this.getNodeParameter('options', i, {});
const node = this.getNode();
options.include = include;
const newItem = await setNode[mode].execute.call(
this,
items[i],
i,
options as SetNodeOptions,
rawData,
node,
);
if (duplicateItem && this.getMode() === 'manual') {
const duplicateCount = this.getNodeParameter('duplicateCount', 0, 0) as number;
for (let j = 0; j <= duplicateCount; j++) {
returnData.push(newItem);
}
} else {
returnData.push(newItem);
}
}
return [returnData];
}
}

View File

@@ -0,0 +1,27 @@
import type { IDataObject } from 'n8n-workflow';
export type SetNodeOptions = {
dotNotation?: boolean;
ignoreConversionErrors?: boolean;
include?: IncludeMods;
includeBinary?: boolean;
};
export type SetField = {
name: string;
type: 'stringValue' | 'numberValue' | 'booleanValue' | 'arrayValue' | 'objectValue';
stringValue?: string;
numberValue?: number;
booleanValue?: boolean;
arrayValue?: string[] | string | IDataObject | IDataObject[];
objectValue?: string | IDataObject;
};
export const INCLUDE = {
ALL: 'all',
NONE: 'none',
SELECTED: 'selected',
EXCEPT: 'except',
} as const;
export type IncludeMods = (typeof INCLUDE)[keyof typeof INCLUDE];

View File

@@ -0,0 +1,211 @@
import type {
FieldType,
IDataObject,
IExecuteFunctions,
INode,
INodeExecutionData,
} from 'n8n-workflow';
import { deepCopy, NodeOperationError, jsonParse, validateFieldType } from 'n8n-workflow';
import set from 'lodash/set';
import get from 'lodash/get';
import unset from 'lodash/unset';
import type { SetNodeOptions, SetField } from './interfaces';
import { INCLUDE } from './interfaces';
import { getResolvables } from '../../../../utils/utilities';
const configureFieldHelper = (dotNotation?: boolean) => {
if (dotNotation !== false) {
return {
set: (item: IDataObject, key: string, value: IDataObject) => {
set(item, key, value);
},
get: (item: IDataObject, key: string) => {
return get(item, key);
},
unset: (item: IDataObject, key: string) => {
unset(item, key);
},
};
} else {
return {
set: (item: IDataObject, key: string, value: IDataObject) => {
item[key] = value;
},
get: (item: IDataObject, key: string) => {
return item[key];
},
unset: (item: IDataObject, key: string) => {
delete item[key];
},
};
}
};
export function composeReturnItem(
this: IExecuteFunctions,
itemIndex: number,
inputItem: INodeExecutionData,
newFields: IDataObject,
options: SetNodeOptions,
) {
const newItem: INodeExecutionData = {
json: {},
pairedItem: inputItem.pairedItem,
};
if (options.includeBinary && inputItem.binary !== undefined) {
// Create a shallow copy of the binary data so that the old
// data references which do not get changed still stay behind
// but the incoming data does not get changed.
newItem.binary = {};
Object.assign(newItem.binary, inputItem.binary);
}
const fieldHelper = configureFieldHelper(options.dotNotation);
switch (options.include) {
case INCLUDE.ALL:
newItem.json = deepCopy(inputItem.json);
break;
case INCLUDE.SELECTED:
const includeFields = (this.getNodeParameter('includeFields', itemIndex) as string)
.split(',')
.map((item) => item.trim())
.filter((item) => item);
for (const key of includeFields) {
const fieldValue = fieldHelper.get(inputItem.json, key) as IDataObject;
let keyToSet = key;
if (options.dotNotation !== false && key.includes('.')) {
keyToSet = key.split('.').pop() as string;
}
fieldHelper.set(newItem.json, keyToSet, fieldValue);
}
break;
case INCLUDE.EXCEPT:
const excludeFields = (this.getNodeParameter('excludeFields', itemIndex) as string)
.split(',')
.map((item) => item.trim())
.filter((item) => item);
const inputData = deepCopy(inputItem.json);
for (const key of excludeFields) {
fieldHelper.unset(inputData, key);
}
newItem.json = inputData;
break;
case INCLUDE.NONE:
break;
default:
throw new Error(`The include option "${options.include}" is not known!`);
}
for (const key of Object.keys(newFields)) {
fieldHelper.set(newItem.json, key, newFields[key] as IDataObject);
}
return newItem;
}
export const parseJsonParameter = (
jsonData: string | IDataObject,
node: INode,
i: number,
entryName?: string,
) => {
let returnData: IDataObject;
const location = entryName ? `entry "${entryName}" inside 'Fields to Set'` : "'JSON Output'";
if (typeof jsonData === 'string') {
try {
returnData = jsonParse<IDataObject>(jsonData);
} catch (error) {
let recoveredData = '';
try {
recoveredData = jsonData
.replace(/'/g, '"') // Replace single quotes with double quotes
.replace(/(['"])?([a-zA-Z0-9_]+)(['"])?:/g, '"$2":') // Wrap keys in double quotes
.replace(/,\s*([\]}])/g, '$1') // Remove trailing commas from objects
.replace(/,+$/, ''); // Remove trailing comma
returnData = jsonParse<IDataObject>(recoveredData);
} catch (err) {
const description =
recoveredData === jsonData ? jsonData : `${recoveredData};\n Original input: ${jsonData}`;
throw new NodeOperationError(node, `The ${location} in item ${i} contains invalid JSON`, {
description,
});
}
}
} else {
returnData = jsonData;
}
if (returnData === undefined || typeof returnData !== 'object' || Array.isArray(returnData)) {
throw new NodeOperationError(
node,
`The ${location} in item ${i} does not contain a valid JSON object`,
);
}
return returnData;
};
export const validateEntry = (
entry: SetField,
node: INode,
itemIndex: number,
ignoreErrors = false,
) => {
let entryValue = entry[entry.type];
const name = entry.name;
const entryType = entry.type.replace('Value', '') as FieldType;
if (entryType === 'string') {
if (typeof entryValue === 'object') {
entryValue = JSON.stringify(entryValue);
} else {
entryValue = String(entryValue);
}
}
const validationResult = validateFieldType(name, entryValue, entryType);
if (!validationResult.valid) {
if (ignoreErrors) {
validationResult.newValue = entry[entry.type];
} else {
const message = `${validationResult.errorMessage} [item ${itemIndex}]`;
const description = `To fix the error try to change the type for the field "${name}" or activate the option “Ignore Type Conversion Errors” to apply a less strict type validation`;
throw new NodeOperationError(node, message, {
itemIndex,
description,
});
}
}
const value = validationResult.newValue === undefined ? null : validationResult.newValue;
return { name, value };
};
export function resolveRawData(this: IExecuteFunctions, rawData: string, i: number) {
const resolvables = getResolvables(rawData);
let returnData: string = rawData;
if (resolvables.length) {
for (const resolvable of resolvables) {
const resolvedValue = this.evaluateExpression(`${resolvable}`, i);
if (typeof resolvedValue === 'object' && resolvedValue !== null) {
returnData = returnData.replace(resolvable, JSON.stringify(resolvedValue));
} else {
returnData = returnData.replace(resolvable, resolvedValue as string);
}
}
}
return returnData;
}

View File

@@ -0,0 +1,208 @@
import type {
IDataObject,
IExecuteFunctions,
INode,
INodeExecutionData,
INodeProperties,
} from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import {
parseJsonParameter,
validateEntry,
composeReturnItem,
resolveRawData,
} from './helpers/utils';
import type { SetField, SetNodeOptions } from './helpers/interfaces';
import { updateDisplayOptions } from '../../../utils/utilities';
const properties: INodeProperties[] = [
{
displayName: 'Fields to Set',
name: 'fields',
placeholder: 'Add Field',
type: 'fixedCollection',
description: 'Edit existing fields or add new ones to modify the output data',
typeOptions: {
multipleValues: true,
sortable: true,
},
default: {},
options: [
{
name: 'values',
displayName: 'Values',
values: [
{
displayName: 'Name',
name: 'name',
type: 'string',
default: '',
placeholder: 'e.g. fieldName',
description:
'Name of the field to set the value of. Supports dot-notation. Example: data.person[0].name.',
requiresDataPath: 'single',
},
{
displayName: 'Type',
name: 'type',
type: 'options',
description: 'The field value type',
// eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items
options: [
{
name: 'String',
value: 'stringValue',
},
{
name: 'Number',
value: 'numberValue',
},
{
name: 'Boolean',
value: 'booleanValue',
},
{
name: 'Array',
value: 'arrayValue',
},
{
name: 'Object',
value: 'objectValue',
},
],
default: 'stringValue',
},
{
displayName: 'Value',
name: 'stringValue',
type: 'string',
default: '',
displayOptions: {
show: {
type: ['stringValue'],
},
},
validateType: 'string',
ignoreValidationDuringExecution: true,
},
{
displayName: 'Value',
name: 'numberValue',
type: 'string',
default: '',
displayOptions: {
show: {
type: ['numberValue'],
},
},
validateType: 'number',
ignoreValidationDuringExecution: true,
},
{
displayName: 'Value',
name: 'booleanValue',
type: 'options',
default: 'true',
options: [
{
name: 'True',
value: 'true',
},
{
name: 'False',
value: 'false',
},
],
displayOptions: {
show: {
type: ['booleanValue'],
},
},
validateType: 'boolean',
ignoreValidationDuringExecution: true,
},
{
displayName: 'Value',
name: 'arrayValue',
type: 'string',
default: '',
placeholder: 'e.g. [ arrayItem1, arrayItem2, arrayItem3 ]',
displayOptions: {
show: {
type: ['arrayValue'],
},
},
validateType: 'array',
ignoreValidationDuringExecution: true,
},
{
displayName: 'Value',
name: 'objectValue',
type: 'string',
default: '={}',
typeOptions: {
editor: 'json',
editorLanguage: 'json',
rows: 2,
},
displayOptions: {
show: {
type: ['objectValue'],
},
},
validateType: 'object',
ignoreValidationDuringExecution: true,
},
],
},
],
},
];
const displayOptions = {
show: {
mode: ['manual'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(
this: IExecuteFunctions,
item: INodeExecutionData,
i: number,
options: SetNodeOptions,
rawFieldsData: IDataObject,
node: INode,
) {
try {
const fields = this.getNodeParameter('fields.values', i, []) as SetField[];
const newData: IDataObject = {};
for (const entry of fields) {
if (entry.type === 'objectValue' && rawFieldsData[entry.name] !== undefined) {
entry.objectValue = parseJsonParameter(
resolveRawData.call(this, rawFieldsData[entry.name] as string, i),
node,
i,
entry.name,
);
}
const { name, value } = validateEntry(entry, node, i, options.ignoreConversionErrors);
newData[name] = value;
}
return composeReturnItem.call(this, i, item, newData, options);
} catch (error) {
if (this.continueOnFail()) {
return { json: { error: (error as Error).message } };
}
throw new NodeOperationError(this.getNode(), error as Error, {
itemIndex: i,
description: error.description,
});
}
}

View File

@@ -0,0 +1,69 @@
import type {
INodeExecutionData,
IExecuteFunctions,
INodeProperties,
IDataObject,
INode,
} from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import { parseJsonParameter, composeReturnItem, resolveRawData } from './helpers/utils';
import type { SetNodeOptions } from './helpers/interfaces';
import { updateDisplayOptions } from '../../../utils/utilities';
const properties: INodeProperties[] = [
{
displayName: 'JSON Output',
name: 'jsonOutput',
type: 'string',
typeOptions: {
editor: 'json',
editorLanguage: 'json',
rows: 5,
},
default: '{\n "my_field_1": "value",\n "my_field_2": 1\n}',
validateType: 'object',
ignoreValidationDuringExecution: true,
},
];
const displayOptions = {
show: {
mode: ['raw'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(
this: IExecuteFunctions,
item: INodeExecutionData,
i: number,
options: SetNodeOptions,
rawData: IDataObject,
node: INode,
) {
try {
let newData: IDataObject;
if (rawData.jsonOutput === undefined) {
const json = this.getNodeParameter('jsonOutput', i) as string;
newData = parseJsonParameter(json, node, i);
} else {
newData = parseJsonParameter(
resolveRawData.call(this, rawData.jsonOutput as string, i),
node,
i,
);
}
return composeReturnItem.call(this, i, item, newData, options);
} catch (error) {
if (this.continueOnFail()) {
return { json: { error: (error as Error).message } };
}
throw new NodeOperationError(node, error as Error, {
itemIndex: i,
description: error.description,
});
}
}