feat: Data transformation nodes and actions in Nodes Panel (#7760)
- Split Items List node into separate nodes per action - Review node descriptions - New icons - New sections in subcategories --------- Co-authored-by: Giulio Andreini <andreini@netseven.it> Co-authored-by: Deborah <deborah@starfallprojects.co.uk> Co-authored-by: Michael Kret <michael.k@radency.com>
This commit is contained in:
Elias Meire
GitHub
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"node": "n8n-nodes-base.removeDuplicates",
|
||||
"nodeVersion": "1.0",
|
||||
"codexVersion": "1.0",
|
||||
"details": "",
|
||||
"categories": ["Core Nodes"],
|
||||
"resources": {
|
||||
"primaryDocumentation": [
|
||||
{
|
||||
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.removeduplicates/"
|
||||
}
|
||||
],
|
||||
"generic": []
|
||||
},
|
||||
"alias": [
|
||||
"Dedupe",
|
||||
"Deduplicate",
|
||||
"Duplicates",
|
||||
"Remove",
|
||||
"Unique",
|
||||
"Transform",
|
||||
"Array",
|
||||
"List",
|
||||
"Item"
|
||||
],
|
||||
"subcategories": {
|
||||
"Core Nodes": ["Data Transformation"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
import get from 'lodash/get';
|
||||
import isEqual from 'lodash/isEqual';
|
||||
import lt from 'lodash/lt';
|
||||
import pick from 'lodash/pick';
|
||||
import {
|
||||
NodeOperationError,
|
||||
type IExecuteFunctions,
|
||||
type INodeExecutionData,
|
||||
type INodeType,
|
||||
type INodeTypeDescription,
|
||||
} from 'n8n-workflow';
|
||||
import { prepareFieldsArray } from '../utils/utils';
|
||||
import { compareItems, flattenKeys } from './utils';
|
||||
|
||||
export class RemoveDuplicates implements INodeType {
|
||||
description: INodeTypeDescription = {
|
||||
displayName: 'Remove Duplicates',
|
||||
name: 'removeDuplicates',
|
||||
icon: 'file:removeDuplicates.svg',
|
||||
group: ['transform'],
|
||||
subtitle: '',
|
||||
version: 1,
|
||||
description: 'Delete items with matching field values',
|
||||
defaults: {
|
||||
name: 'Remove Duplicates',
|
||||
},
|
||||
inputs: ['main'],
|
||||
outputs: ['main'],
|
||||
properties: [
|
||||
{
|
||||
displayName: 'Compare',
|
||||
name: 'compare',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'All Fields',
|
||||
value: 'allFields',
|
||||
},
|
||||
{
|
||||
name: 'All Fields Except',
|
||||
value: 'allFieldsExcept',
|
||||
},
|
||||
{
|
||||
name: 'Selected Fields',
|
||||
value: 'selectedFields',
|
||||
},
|
||||
],
|
||||
default: 'allFields',
|
||||
description: 'The fields of the input items to compare to see if they are the same',
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Exclude',
|
||||
name: 'fieldsToExclude',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. email, name',
|
||||
requiresDataPath: 'multiple',
|
||||
description: 'Fields in the input to exclude from the comparison',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
compare: ['allFieldsExcept'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields To Compare',
|
||||
name: 'fieldsToCompare',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. email, name',
|
||||
requiresDataPath: 'multiple',
|
||||
description: 'Fields in the input to add to the comparison',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
compare: ['selectedFields'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add Field',
|
||||
default: {},
|
||||
displayOptions: {
|
||||
show: {
|
||||
compare: ['allFieldsExcept', 'selectedFields'],
|
||||
},
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Disable Dot Notation',
|
||||
name: 'disableDotNotation',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to disallow referencing child fields using `parent.child` in the field name',
|
||||
},
|
||||
{
|
||||
displayName: 'Remove Other Fields',
|
||||
name: 'removeOtherFields',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
const items = this.getInputData();
|
||||
const compare = this.getNodeParameter('compare', 0) as string;
|
||||
const disableDotNotation = this.getNodeParameter(
|
||||
'options.disableDotNotation',
|
||||
0,
|
||||
false,
|
||||
) as boolean;
|
||||
const removeOtherFields = this.getNodeParameter(
|
||||
'options.removeOtherFields',
|
||||
0,
|
||||
false,
|
||||
) as boolean;
|
||||
|
||||
let keys = disableDotNotation
|
||||
? Object.keys(items[0].json)
|
||||
: Object.keys(flattenKeys(items[0].json));
|
||||
|
||||
for (const item of items) {
|
||||
for (const key of disableDotNotation
|
||||
? Object.keys(item.json)
|
||||
: Object.keys(flattenKeys(item.json))) {
|
||||
if (!keys.includes(key)) {
|
||||
keys.push(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (compare === 'allFieldsExcept') {
|
||||
const fieldsToExclude = prepareFieldsArray(
|
||||
this.getNodeParameter('fieldsToExclude', 0, '') as string,
|
||||
'Fields To Exclude',
|
||||
);
|
||||
|
||||
if (!fieldsToExclude.length) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
'No fields specified. Please add a field to exclude from comparison',
|
||||
);
|
||||
}
|
||||
if (!disableDotNotation) {
|
||||
keys = Object.keys(flattenKeys(items[0].json));
|
||||
}
|
||||
keys = keys.filter((key) => !fieldsToExclude.includes(key));
|
||||
}
|
||||
if (compare === 'selectedFields') {
|
||||
const fieldsToCompare = prepareFieldsArray(
|
||||
this.getNodeParameter('fieldsToCompare', 0, '') as string,
|
||||
'Fields To Compare',
|
||||
);
|
||||
if (!fieldsToCompare.length) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
'No fields specified. Please add a field to compare on',
|
||||
);
|
||||
}
|
||||
if (!disableDotNotation) {
|
||||
keys = Object.keys(flattenKeys(items[0].json));
|
||||
}
|
||||
keys = fieldsToCompare.map((key) => key.trim());
|
||||
}
|
||||
|
||||
// This solution is O(nlogn)
|
||||
// add original index to the items
|
||||
const newItems = items.map(
|
||||
(item, index) =>
|
||||
({
|
||||
json: { ...item.json, __INDEX: index },
|
||||
pairedItem: { item: index },
|
||||
}) as INodeExecutionData,
|
||||
);
|
||||
//sort items using the compare keys
|
||||
newItems.sort((a, b) => {
|
||||
let result = 0;
|
||||
|
||||
for (const key of keys) {
|
||||
let equal;
|
||||
if (!disableDotNotation) {
|
||||
equal = isEqual(get(a.json, key), get(b.json, key));
|
||||
} else {
|
||||
equal = isEqual(a.json[key], b.json[key]);
|
||||
}
|
||||
if (!equal) {
|
||||
let lessThan;
|
||||
if (!disableDotNotation) {
|
||||
lessThan = lt(get(a.json, key), get(b.json, key));
|
||||
} else {
|
||||
lessThan = lt(a.json[key], b.json[key]);
|
||||
}
|
||||
result = lessThan ? -1 : 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
});
|
||||
|
||||
for (const key of keys) {
|
||||
let type: any = undefined;
|
||||
for (const item of newItems) {
|
||||
if (key === '') {
|
||||
throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank');
|
||||
}
|
||||
const value = !disableDotNotation ? get(item.json, key) : item.json[key];
|
||||
if (value === undefined && disableDotNotation && key.includes('.')) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`'${key}' field is missing from some input items`,
|
||||
{
|
||||
description:
|
||||
"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
|
||||
},
|
||||
);
|
||||
} else if (value === undefined) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`'${key}' field is missing from some input items`,
|
||||
);
|
||||
}
|
||||
if (type !== undefined && value !== undefined && type !== typeof value) {
|
||||
throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, {
|
||||
description: 'The type of this field varies between items',
|
||||
});
|
||||
} else {
|
||||
type = typeof value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// collect the original indexes of items to be removed
|
||||
const removedIndexes: number[] = [];
|
||||
let temp = newItems[0];
|
||||
for (let index = 1; index < newItems.length; index++) {
|
||||
if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
|
||||
removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
|
||||
} else {
|
||||
temp = newItems[index];
|
||||
}
|
||||
}
|
||||
|
||||
let returnData = items.filter((_, index) => !removedIndexes.includes(index));
|
||||
|
||||
if (removeOtherFields) {
|
||||
returnData = returnData.map((item, index) => ({
|
||||
json: pick(item.json, ...keys),
|
||||
pairedItem: { item: index },
|
||||
}));
|
||||
}
|
||||
|
||||
return [returnData];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<svg width="512" height="512" viewBox="0 0 512 512" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_1156_1098)">
|
||||
<path d="M134.097 111H172.926V143.508H138.16V178.143H105.652V139.444C105.652 123.735 118.387 111 134.097 111Z" fill="#54B8C9"/>
|
||||
<path d="M211.755 143.508V111H289.412V143.508H211.755Z" fill="#54B8C9"/>
|
||||
<path d="M328.241 143.508V111H405.899V143.508H328.241Z" fill="#54B8C9"/>
|
||||
<path d="M444.728 143.508V111H483.557C499.267 111 512.002 123.735 512.002 139.444V178.143H479.494V143.508H444.728Z" fill="#54B8C9"/>
|
||||
<path d="M479.494 216.746H512.002V255.444C512.002 271.154 499.267 283.889 483.557 283.889H444.728V251.381H479.494V216.746Z" fill="#54B8C9"/>
|
||||
<path d="M0 244.537C0 229.329 12.735 217 28.4444 217H377.905C393.614 217 406.349 229.329 406.349 244.537V374.352C406.349 389.56 393.614 401.889 377.905 401.889H28.4444C12.735 401.889 0 389.56 0 374.352V244.537Z" fill="#54B8C9"/>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id="clip0_1156_1098">
|
||||
<rect width="512" height="512" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.0 KiB |
@@ -0,0 +1,5 @@
|
||||
import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers';
|
||||
|
||||
const workflows = getWorkflowFilenames(__dirname);
|
||||
|
||||
describe('Test Remove Duplicates Node', () => testWorkflows(workflows));
|
||||
@@ -0,0 +1,326 @@
|
||||
{
|
||||
"name": "Remove Duplicates",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "a4da10da-991f-48ab-b873-9d633a11311f",
|
||||
"name": "When clicking \"Execute Workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
760,
|
||||
420
|
||||
]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"jsCode": "return [{ id: 1, name: 'John Doe', age: 18 },{ id: 1, name: 'John Doe', age: 18 },\n { id: 1, name: 'John Doe', age: 98 },\n { id: 3, name: 'Bob Johnson', age:34 }]"
|
||||
},
|
||||
"id": "7ab7d5cd-0b1e-48bc-bdbd-57c91e201cf3",
|
||||
"name": "Code",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 2,
|
||||
"position": [
|
||||
980,
|
||||
420
|
||||
]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "c336939c-062e-475e-ba7c-8601e3662e8c",
|
||||
"name": "Remove Duplicates (All Fields)",
|
||||
"type": "n8n-nodes-base.removeDuplicates",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
1200,
|
||||
260
|
||||
]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"compare": "selectedFields",
|
||||
"fieldsToCompare": "name",
|
||||
"options": {}
|
||||
},
|
||||
"id": "d4343ffe-8a9e-4e34-a0a1-aa463afedd80",
|
||||
"name": "Remove Duplicates (Selected Fields)",
|
||||
"type": "n8n-nodes-base.removeDuplicates",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
1200,
|
||||
420
|
||||
]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"compare": "allFieldsExcept",
|
||||
"fieldsToExclude": "age",
|
||||
"options": {}
|
||||
},
|
||||
"id": "b67daea4-4545-429e-9e2a-58f2d6a7df7b",
|
||||
"name": "Remove Duplicates (Except Fields)",
|
||||
"type": "n8n-nodes-base.removeDuplicates",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
1200,
|
||||
580
|
||||
]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "813e690f-a83e-4a38-a64a-c3d72afcc9ba",
|
||||
"name": "All Fields",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
1380,
|
||||
260
|
||||
]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "b5c5c946-2e96-451b-b9a6-78e478504d6c",
|
||||
"name": "Selected Fields",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
1380,
|
||||
420
|
||||
]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "afb92bc5-beba-4b0a-aefb-b47cc708a125",
|
||||
"name": "Except Fields",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
1380,
|
||||
580
|
||||
]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"compare": "allFieldsExcept",
|
||||
"fieldsToExclude": "age",
|
||||
"options": {
|
||||
"removeOtherFields": true
|
||||
}
|
||||
},
|
||||
"id": "f92c5533-ac29-476c-aebb-4849ddd22110",
|
||||
"name": "Remove Duplicates (Remove)",
|
||||
"type": "n8n-nodes-base.removeDuplicates",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
1200,
|
||||
760
|
||||
]
|
||||
},
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "1e142ab7-b32e-4f67-b5cc-5c9fb63fba89",
|
||||
"name": "Remove",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
1380,
|
||||
760
|
||||
]
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"Code": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 98
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson",
|
||||
"age": 34
|
||||
}
|
||||
}
|
||||
],
|
||||
"All Fields": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 98
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson",
|
||||
"age": 34
|
||||
}
|
||||
}
|
||||
],
|
||||
"Selected Fields": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson",
|
||||
"age": 34
|
||||
}
|
||||
}
|
||||
],
|
||||
"Except Fields": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe",
|
||||
"age": 18
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson",
|
||||
"age": 34
|
||||
}
|
||||
}
|
||||
],
|
||||
"Remove": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"name": "John Doe"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"name": "Bob Johnson"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute Workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Code",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Code": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Remove Duplicates (All Fields)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Remove Duplicates (Selected Fields)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Remove Duplicates (Except Fields)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
},
|
||||
{
|
||||
"node": "Remove Duplicates (Remove)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Remove Duplicates (All Fields)": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "All Fields",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Remove Duplicates (Selected Fields)": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Selected Fields",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Remove Duplicates (Except Fields)": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Except Fields",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Remove Duplicates (Remove)": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Remove",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {
|
||||
"executionOrder": "v1"
|
||||
},
|
||||
"versionId": "5bb09766-4c67-4fb4-ae53-89d8db4727e3",
|
||||
"id": "74gMYOHjjPArZg4q",
|
||||
"meta": {
|
||||
"instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4"
|
||||
},
|
||||
"tags": [
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import get from 'lodash/get';
|
||||
import isEqual from 'lodash/isEqual';
|
||||
import isObject from 'lodash/isObject';
|
||||
import merge from 'lodash/merge';
|
||||
import reduce from 'lodash/reduce';
|
||||
import type { IDataObject, INode, INodeExecutionData } from 'n8n-workflow';
|
||||
|
||||
export const compareItems = (
|
||||
obj: INodeExecutionData,
|
||||
obj2: INodeExecutionData,
|
||||
keys: string[],
|
||||
disableDotNotation: boolean,
|
||||
_node: INode,
|
||||
) => {
|
||||
let result = true;
|
||||
for (const key of keys) {
|
||||
if (!disableDotNotation) {
|
||||
if (!isEqual(get(obj.json, key), get(obj2.json, key))) {
|
||||
result = false;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (!isEqual(obj.json[key], obj2.json[key])) {
|
||||
result = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => {
|
||||
return !isObject(obj)
|
||||
? { [path.join('.')]: obj }
|
||||
: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
|
||||
};
|
||||
Reference in New Issue
Block a user