feat: Data transformation nodes and actions in Nodes Panel (#7760)

- Split Items List node into separate nodes per action
- Review node descriptions
- New icons
- New sections in subcategories

---------

Co-authored-by: Giulio Andreini <andreini@netseven.it>
Co-authored-by: Deborah <deborah@starfallprojects.co.uk>
Co-authored-by: Michael Kret <michael.k@radency.com>
This commit is contained in:
Elias Meire
2023-12-08 11:40:05 +01:00
committed by GitHub
parent 90824b50ed
commit 675ec21d33
78 changed files with 4353 additions and 74 deletions

View File

@@ -0,0 +1,19 @@
{
"node": "n8n-nodes-base.aggregate",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"details": "",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.aggregate/"
}
],
"generic": []
},
"alias": ["Aggregate", "Combine", "Flatten", "Transform", "Array", "List", "Item"],
"subcategories": {
"Core Nodes": ["Data Transformation"]
}
}

View File

@@ -0,0 +1,414 @@
import get from 'lodash/get';
import isEmpty from 'lodash/isEmpty';
import set from 'lodash/set';
import {
NodeOperationError,
type IDataObject,
type IExecuteFunctions,
type INodeExecutionData,
type INodeType,
type INodeTypeDescription,
type IPairedItemData,
} from 'n8n-workflow';
import { prepareFieldsArray } from '../utils/utils';
import { addBinariesToItem } from './utils';
export class Aggregate implements INodeType {
description: INodeTypeDescription = {
displayName: 'Aggregate',
name: 'aggregate',
icon: 'file:aggregate.svg',
group: ['transform'],
subtitle: '',
version: 1,
description: 'Combine a field from many items into a list in a single item',
defaults: {
name: 'Aggregate',
},
inputs: ['main'],
outputs: ['main'],
properties: [
{
displayName: 'Aggregate',
name: 'aggregate',
type: 'options',
default: 'aggregateIndividualFields',
options: [
{
name: 'Individual Fields',
value: 'aggregateIndividualFields',
},
{
name: 'All Item Data (Into a Single List)',
value: 'aggregateAllItemData',
},
],
},
{
displayName: 'Fields To Aggregate',
name: 'fieldsToAggregate',
type: 'fixedCollection',
typeOptions: {
multipleValues: true,
},
placeholder: 'Add Field To Aggregate',
default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] },
displayOptions: {
show: {
aggregate: ['aggregateIndividualFields'],
},
},
options: [
{
displayName: '',
name: 'fieldToAggregate',
values: [
{
displayName: 'Input Field Name',
name: 'fieldToAggregate',
type: 'string',
default: '',
description: 'The name of a field in the input items to aggregate together',
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
placeholder: 'e.g. id',
hint: ' Enter the field name as text',
requiresDataPath: 'single',
},
{
displayName: 'Rename Field',
name: 'renameField',
type: 'boolean',
default: false,
description: 'Whether to give the field a different name in the output',
},
{
displayName: 'Output Field Name',
name: 'outputFieldName',
displayOptions: {
show: {
renameField: [true],
},
},
type: 'string',
default: '',
description:
'The name of the field to put the aggregated data in. Leave blank to use the input field name.',
requiresDataPath: 'single',
},
],
},
],
},
{
displayName: 'Put Output in Field',
name: 'destinationFieldName',
type: 'string',
displayOptions: {
show: {
aggregate: ['aggregateAllItemData'],
},
},
default: 'data',
description: 'The name of the output field to put the data in',
},
{
displayName: 'Include',
name: 'include',
type: 'options',
default: 'allFields',
options: [
{
name: 'All Fields',
value: 'allFields',
},
{
name: 'Specified Fields',
value: 'specifiedFields',
},
{
name: 'All Fields Except',
value: 'allFieldsExcept',
},
],
displayOptions: {
show: {
aggregate: ['aggregateAllItemData'],
},
},
},
{
displayName: 'Fields To Exclude',
name: 'fieldsToExclude',
type: 'string',
placeholder: 'e.g. email, name',
default: '',
requiresDataPath: 'multiple',
displayOptions: {
show: {
aggregate: ['aggregateAllItemData'],
include: ['allFieldsExcept'],
},
},
},
{
displayName: 'Fields To Include',
name: 'fieldsToInclude',
type: 'string',
placeholder: 'e.g. email, name',
default: '',
requiresDataPath: 'multiple',
displayOptions: {
show: {
aggregate: ['aggregateAllItemData'],
include: ['specifiedFields'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Field',
default: {},
options: [
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
displayOptions: {
hide: {
'/aggregate': ['aggregateAllItemData'],
},
},
},
{
displayName: 'Merge Lists',
name: 'mergeLists',
type: 'boolean',
default: false,
description:
'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list',
displayOptions: {
hide: {
'/aggregate': ['aggregateAllItemData'],
},
},
},
{
displayName: 'Include Binaries',
name: 'includeBinaries',
type: 'boolean',
default: false,
description: 'Whether to include the binary data in the new item',
},
{
displayName: 'Keep Only Unique Binaries',
name: 'keepOnlyUnique',
type: 'boolean',
default: false,
description:
'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions',
displayOptions: {
show: {
includeBinaries: [true],
},
},
},
{
displayName: 'Keep Missing And Null Values',
name: 'keepMissing',
type: 'boolean',
default: false,
description:
'Whether to add a null entry to the aggregated list when there is a missing or null value',
displayOptions: {
hide: {
'/aggregate': ['aggregateAllItemData'],
},
},
},
],
},
],
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
let returnData: INodeExecutionData = { json: {}, pairedItem: [] };
const items = this.getInputData();
const aggregate = this.getNodeParameter('aggregate', 0, '') as string;
if (aggregate === 'aggregateIndividualFields') {
const disableDotNotation = this.getNodeParameter(
'options.disableDotNotation',
0,
false,
) as boolean;
const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean;
const fieldsToAggregate = this.getNodeParameter(
'fieldsToAggregate.fieldToAggregate',
0,
[],
) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }];
const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean;
if (!fieldsToAggregate.length) {
throw new NodeOperationError(this.getNode(), 'No fields specified', {
description: 'Please add a field to aggregate',
});
}
const newItem: INodeExecutionData = {
json: {},
pairedItem: Array.from({ length: items.length }, (_, i) => i).map((index) => {
return {
item: index,
};
}),
};
const values: { [key: string]: any } = {};
const outputFields: string[] = [];
for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) {
const field = renameField ? outputFieldName : fieldToAggregate;
if (outputFields.includes(field)) {
throw new NodeOperationError(
this.getNode(),
`The '${field}' output field is used more than once`,
{ description: 'Please make sure each output field name is unique' },
);
} else {
outputFields.push(field);
}
const getFieldToAggregate = () =>
!disableDotNotation && fieldToAggregate.includes('.')
? fieldToAggregate.split('.').pop()
: fieldToAggregate;
const _outputFieldName = outputFieldName
? outputFieldName
: (getFieldToAggregate() as string);
if (fieldToAggregate !== '') {
values[_outputFieldName] = [];
for (let i = 0; i < items.length; i++) {
if (!disableDotNotation) {
let value = get(items[i].json, fieldToAggregate);
if (!keepMissing) {
if (Array.isArray(value)) {
value = value.filter((entry) => entry !== null);
} else if (value === null || value === undefined) {
continue;
}
}
if (Array.isArray(value) && mergeLists) {
values[_outputFieldName].push(...value);
} else {
values[_outputFieldName].push(value);
}
} else {
let value = items[i].json[fieldToAggregate];
if (!keepMissing) {
if (Array.isArray(value)) {
value = value.filter((entry) => entry !== null);
} else if (value === null || value === undefined) {
continue;
}
}
if (Array.isArray(value) && mergeLists) {
values[_outputFieldName].push(...value);
} else {
values[_outputFieldName].push(value);
}
}
}
}
}
for (const key of Object.keys(values)) {
if (!disableDotNotation) {
set(newItem.json, key, values[key]);
} else {
newItem.json[key] = values[key];
}
}
returnData = newItem;
} else {
let newItems: IDataObject[] = items.map((item) => item.json);
let pairedItem: IPairedItemData[] = [];
const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string;
const fieldsToExclude = prepareFieldsArray(
this.getNodeParameter('fieldsToExclude', 0, '') as string,
'Fields To Exclude',
);
const fieldsToInclude = prepareFieldsArray(
this.getNodeParameter('fieldsToInclude', 0, '') as string,
'Fields To Include',
);
if (fieldsToExclude.length || fieldsToInclude.length) {
newItems = newItems.reduce((acc, item, index) => {
const newItem: IDataObject = {};
let outputFields = Object.keys(item);
if (fieldsToExclude.length) {
outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key));
}
if (fieldsToInclude.length) {
outputFields = outputFields.filter((key) =>
fieldsToInclude.length ? fieldsToInclude.includes(key) : true,
);
}
outputFields.forEach((key) => {
newItem[key] = item[key];
});
if (isEmpty(newItem)) {
return acc;
}
pairedItem.push({ item: index });
return acc.concat([newItem]);
}, [] as IDataObject[]);
} else {
pairedItem = Array.from({ length: newItems.length }, (_, item) => ({
item,
}));
}
const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem };
returnData = output;
}
const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean;
if (includeBinaries) {
const pairedItems = (returnData.pairedItem || []) as IPairedItemData[];
const aggregatedItems = pairedItems.map((item) => {
return items[item.item];
});
const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean;
addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique);
}
return [[returnData]];
}
}

View File

@@ -0,0 +1,14 @@
<svg width="512" height="512" viewBox="0 0 512 512" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_1147_318)">
<path fill-rule="evenodd" clip-rule="evenodd" d="M32 148C32 141.373 37.3726 136 44 136L190 136C196.627 136 202 141.373 202 148L202 172C202 178.627 196.627 184 190 184L44 184C37.3726 184 32 178.627 32 172L32 148Z" fill="#FF6D5A"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M32 244C32 237.373 37.3726 232 44 232L190 232C196.627 232 202 237.373 202 244L202 268C202 274.627 196.627 280 190 280L44 280C37.3726 280 32 274.627 32 268L32 244Z" fill="#FF6D5A"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M32 340C32 333.373 37.3726 328 44 328L190 328C196.627 328 202 333.373 202 340L202 364C202 370.627 196.627 376 190 376L44 376C37.3726 376 32 370.627 32 364L32 340Z" fill="#FF6D5A"/>
<path d="M74 76C74 82.6274 79.3726 88 86 88H202.217C219.89 88 234.217 102.327 234.217 120V176C234.217 202.978 244.489 227.557 261.336 246.039C266.391 251.584 266.391 260.416 261.336 265.961C244.489 284.443 234.217 309.022 234.217 336V392C234.217 409.673 219.89 424 202.217 424H86C79.3726 424 74 429.373 74 436V460C74 466.627 79.3726 472 86 472H202.217C246.4 472 282.217 436.183 282.217 392V336C282.217 305.072 307.289 280 338.217 280V280C341.411 280 344 277.411 344 274.217V237.783C344 234.589 341.411 232 338.217 232V232C307.289 232 282.217 206.928 282.217 176V120C282.217 75.8172 246.4 40 202.217 40H86C79.3726 40 74 45.3726 74 52V76Z" fill="#FF6D5A"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M376 244C376 237.373 381.373 232 388 232L500 232C506.627 232 512 237.373 512 244L512 268C512 274.627 506.627 280 500 280L388 280C381.373 280 376 274.627 376 268L376 244Z" fill="#FF6D5A"/>
</g>
<defs>
<clipPath id="clip0_1147_318">
<rect width="512" height="512" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 1.8 KiB

View File

@@ -0,0 +1,5 @@
import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers';
const workflows = getWorkflowFilenames(__dirname);
describe('Test Aggregate Node', () => testWorkflows(workflows));

View File

@@ -0,0 +1,207 @@
{
"name": "itemLists test",
"nodes": [
{
"parameters": {},
"id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [-440, 260]
},
{
"parameters": {
"jsCode": "return [\n {id: 1, char: 'a'},\n {id: 2, char: 'b'},\n {id: 3, char: 'c'},\n {id: 4, char: 'd'},\n {id: 5, char: 'e'},\n];"
},
"id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5",
"name": "Code",
"type": "n8n-nodes-base.code",
"typeVersion": 1,
"position": [-180, 260]
},
{
"parameters": {
"fieldsToAggregate": {
"fieldToAggregate": [
{
"fieldToAggregate": "id",
"renameField": true,
"outputFieldName": "data"
}
]
},
"options": {}
},
"id": "d95ca3a3-fb43-4037-846e-b87103dec1a3",
"name": "fields aggregate and rename",
"type": "n8n-nodes-base.aggregate",
"typeVersion": 1,
"position": [80, 0]
},
{
"parameters": {
"aggregate": "aggregateAllItemData"
},
"id": "4c1bc7be-7611-418d-aad5-8642b1cc0781",
"name": "aggregate all fields into list",
"type": "n8n-nodes-base.aggregate",
"typeVersion": 1,
"position": [80, 320]
},
{
"parameters": {
"aggregate": "aggregateAllItemData",
"include": "specifiedFields",
"fieldsToInclude": ["id"]
},
"id": "951de23c-2018-437b-961e-8ae7d7fd1a82",
"name": "aggregate selected fields into list",
"type": "n8n-nodes-base.aggregate",
"typeVersion": 1,
"position": [80, 500]
},
{
"parameters": {
"aggregate": "aggregateAllItemData",
"destinationFieldName": "output",
"include": "allFieldsExcept",
"fieldsToExclude": ["char"]
},
"id": "b62c02ee-5edb-473d-a755-7fb8700641fa",
"name": "aggregate all fields except selected into list",
"type": "n8n-nodes-base.aggregate",
"typeVersion": 1,
"position": [80, 700]
}
],
"pinData": {
"fields aggregate and rename": [
{
"json": {
"data": [1, 2, 3, 4, 5]
}
}
],
"aggregate all fields into list": [
{
"json": {
"data": [
{
"id": 1,
"char": "a"
},
{
"id": 2,
"char": "b"
},
{
"id": 3,
"char": "c"
},
{
"id": 4,
"char": "d"
},
{
"id": 5,
"char": "e"
}
]
}
}
],
"aggregate selected fields into list": [
{
"json": {
"data": [
{
"id": 1
},
{
"id": 2
},
{
"id": 3
},
{
"id": 4
},
{
"id": 5
}
]
}
}
],
"aggregate all fields except selected into list": [
{
"json": {
"output": [
{
"id": 1
},
{
"id": 2
},
{
"id": 3
},
{
"id": 4
},
{
"id": 5
}
]
}
}
]
},
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
},
"Code": {
"main": [
[
{
"node": "fields aggregate and rename",
"type": "main",
"index": 0
},
{
"node": "aggregate all fields into list",
"type": "main",
"index": 0
},
{
"node": "aggregate selected fields into list",
"type": "main",
"index": 0
},
{
"node": "aggregate all fields except selected into list",
"type": "main",
"index": 0
}
]
]
}
},
"active": false,
"settings": {},
"versionId": "9bf7c52b-b118-4dad-bfef-7db41828393b",
"id": "105",
"meta": {
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
},
"tags": []
}

View File

@@ -0,0 +1,60 @@
import type { IBinaryData, INodeExecutionData } from 'n8n-workflow';
type PartialBinaryData = Omit<IBinaryData, 'data'>;
const isBinaryUniqueSetup = () => {
const binaries: PartialBinaryData[] = [];
return (binary: IBinaryData) => {
for (const existingBinary of binaries) {
if (
existingBinary.mimeType === binary.mimeType &&
existingBinary.fileType === binary.fileType &&
existingBinary.fileSize === binary.fileSize &&
existingBinary.fileExtension === binary.fileExtension
) {
return false;
}
}
binaries.push({
mimeType: binary.mimeType,
fileType: binary.fileType,
fileSize: binary.fileSize,
fileExtension: binary.fileExtension,
});
return true;
};
};
export function addBinariesToItem(
newItem: INodeExecutionData,
items: INodeExecutionData[],
uniqueOnly?: boolean,
) {
const isBinaryUnique = uniqueOnly ? isBinaryUniqueSetup() : undefined;
for (const item of items) {
if (item.binary === undefined) continue;
for (const key of Object.keys(item.binary)) {
if (!newItem.binary) newItem.binary = {};
let binaryKey = key;
const binary = item.binary[key];
if (isBinaryUnique && !isBinaryUnique(binary)) {
continue;
}
// If the binary key already exists add a suffix to it
let i = 1;
while (newItem.binary[binaryKey] !== undefined) {
binaryKey = `${key}_${i}`;
i++;
}
newItem.binary[binaryKey] = binary;
}
}
return newItem;
}