feat(Merge Node): Overhaul, v3 (#9528)

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
Co-authored-by: Shireen Missi <94372015+ShireenMissi@users.noreply.github.com>
This commit is contained in:
Michael Kret
2024-07-02 13:47:04 +03:00
committed by GitHub
parent 16b1a094b1
commit af69c80bf5
32 changed files with 2151 additions and 48 deletions

View File

@@ -0,0 +1,31 @@
/* eslint-disable n8n-nodes-base/node-filename-against-convention */
import type {
IExecuteFunctions,
INodeType,
INodeTypeBaseDescription,
INodeTypeDescription,
} from 'n8n-workflow';
import { versionDescription } from './actions/versionDescription';
import { router } from './actions/router';
import { loadOptions } from './methods';
export class MergeV3 implements INodeType {
description: INodeTypeDescription;
constructor(baseDescription: INodeTypeBaseDescription) {
this.description = {
...baseDescription,
...versionDescription,
};
}
methods = {
loadOptions,
};
async execute(this: IExecuteFunctions) {
return await router.call(this);
}
}

View File

@@ -0,0 +1,32 @@
import {
type IExecuteFunctions,
type INodeExecutionData,
type INodeProperties,
} from 'n8n-workflow';
import { updateDisplayOptions } from '@utils/utilities';
import { numberInputsProperty } from '../../helpers/descriptions';
export const properties: INodeProperties[] = [numberInputsProperty];
const displayOptions = {
show: {
mode: ['append'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(
this: IExecuteFunctions,
inputsData: INodeExecutionData[][],
): Promise<INodeExecutionData[]> {
const returnData: INodeExecutionData[] = [];
for (let i = 0; i < inputsData.length; i++) {
returnData.push.apply(returnData, inputsData[i]);
}
return returnData;
}

View File

@@ -0,0 +1,110 @@
import { NodeOperationError } from 'n8n-workflow';
import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
import { preparePairedItemDataArray, updateDisplayOptions } from '@utils/utilities';
import { numberInputsProperty } from '../../helpers/descriptions';
export const properties: INodeProperties[] = [
numberInputsProperty,
{
displayName: 'Output Type',
name: 'chooseBranchMode',
type: 'options',
options: [
{
name: 'Wait for All Inputs to Arrive',
value: 'waitForAll',
},
],
default: 'waitForAll',
},
{
displayName: 'Output',
name: 'output',
type: 'options',
options: [
{
name: 'Data of Specified Input',
value: 'specifiedInput',
},
{
name: 'A Single, Empty Item',
value: 'empty',
},
],
default: 'specifiedInput',
displayOptions: {
show: {
chooseBranchMode: ['waitForAll'],
},
},
},
{
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options
displayName: 'Use Data of Input',
name: 'useDataOfInput',
type: 'options',
default: 1,
displayOptions: {
show: {
output: ['specifiedInput'],
},
},
typeOptions: {
minValue: 1,
loadOptionsMethod: 'getInputs',
loadOptionsDependsOn: ['numberInputs'],
},
// eslint-disable-next-line n8n-nodes-base/node-param-description-wrong-for-dynamic-options
description: 'The number of the input to use data of',
validateType: 'number',
},
];
const displayOptions = {
show: {
mode: ['chooseBranch'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(
this: IExecuteFunctions,
inputsData: INodeExecutionData[][],
): Promise<INodeExecutionData[]> {
const returnData: INodeExecutionData[] = [];
const chooseBranchMode = this.getNodeParameter('chooseBranchMode', 0) as string;
if (chooseBranchMode === 'waitForAll') {
const output = this.getNodeParameter('output', 0) as string;
if (output === 'specifiedInput') {
const useDataOfInput = this.getNodeParameter('useDataOfInput', 0) as number;
if (useDataOfInput > inputsData.length) {
throw new NodeOperationError(this.getNode(), `Input ${useDataOfInput} doesn't exist`, {
description: `The node has only ${inputsData.length} inputs, so selecting input ${useDataOfInput} is not possible.`,
});
}
const inputData = inputsData[useDataOfInput - 1];
returnData.push.apply(returnData, inputData);
}
if (output === 'empty') {
const pairedItem = [
...this.getInputData(0).map((inputData) => inputData.pairedItem),
...this.getInputData(1).map((inputData) => inputData.pairedItem),
].flatMap(preparePairedItemDataArray);
returnData.push({
json: {},
pairedItem,
});
}
}
return returnData;
}

View File

@@ -0,0 +1,84 @@
import type {
IExecuteFunctions,
INodeExecutionData,
INodeProperties,
IPairedItemData,
} from 'n8n-workflow';
import { updateDisplayOptions } from '@utils/utilities';
import type { ClashResolveOptions } from '../../helpers/interfaces';
import { clashHandlingProperties, fuzzyCompareProperty } from '../../helpers/descriptions';
import { addSuffixToEntriesKeys, selectMergeMethod } from '../../helpers/utils';
import merge from 'lodash/merge';
export const properties: INodeProperties[] = [
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [clashHandlingProperties, fuzzyCompareProperty],
},
];
const displayOptions = {
show: {
mode: ['combine'],
combineBy: ['combineAll'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(
this: IExecuteFunctions,
inputsData: INodeExecutionData[][],
): Promise<INodeExecutionData[]> {
const returnData: INodeExecutionData[] = [];
const clashHandling = this.getNodeParameter(
'options.clashHandling.values',
0,
{},
) as ClashResolveOptions;
let input1 = inputsData[0];
let input2 = inputsData[1];
if (clashHandling.resolveClash === 'preferInput1') {
[input1, input2] = [input2, input1];
}
if (clashHandling.resolveClash === 'addSuffix') {
input1 = addSuffixToEntriesKeys(input1, '1');
input2 = addSuffixToEntriesKeys(input2, '2');
}
const mergeIntoSingleObject = selectMergeMethod(clashHandling);
if (!input1 || !input2) {
return returnData;
}
let entry1: INodeExecutionData;
let entry2: INodeExecutionData;
for (entry1 of input1) {
for (entry2 of input2) {
returnData.push({
json: {
...mergeIntoSingleObject(entry1.json, entry2.json),
},
binary: {
...merge({}, entry1.binary, entry2.binary),
},
pairedItem: [entry1.pairedItem as IPairedItemData, entry2.pairedItem as IPairedItemData],
});
}
}
return returnData;
}

View File

@@ -0,0 +1,420 @@
import type {
IDataObject,
IExecuteFunctions,
INodeExecutionData,
INodeProperties,
} from 'n8n-workflow';
import { updateDisplayOptions } from '@utils/utilities';
import type {
ClashResolveOptions,
MatchFieldsJoinMode,
MatchFieldsOptions,
MatchFieldsOutput,
} from '../../helpers/interfaces';
import { clashHandlingProperties, fuzzyCompareProperty } from '../../helpers/descriptions';
import {
addSourceField,
addSuffixToEntriesKeys,
checkInput,
checkMatchFieldsInput,
findMatches,
mergeMatched,
} from '../../helpers/utils';
const multipleMatchesProperty: INodeProperties = {
displayName: 'Multiple Matches',
name: 'multipleMatches',
type: 'options',
default: 'all',
options: [
{
name: 'Include All Matches',
value: 'all',
description: 'Output multiple items if there are multiple matches',
},
{
name: 'Include First Match Only',
value: 'first',
description: 'Only ever output a single item per match',
},
],
};
export const properties: INodeProperties[] = [
{
displayName: 'Fields To Match Have Different Names',
name: 'advanced',
type: 'boolean',
default: false,
description: 'Whether name(s) of field to match are different in input 1 and input 2',
},
{
displayName: 'Fields to Match',
name: 'fieldsToMatchString',
type: 'string',
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
placeholder: 'e.g. id, name',
default: '',
requiresDataPath: 'multiple',
description: 'Specify the fields to use for matching input items',
hint: 'Drag or type the input field name',
displayOptions: {
show: {
advanced: [false],
},
},
},
{
displayName: 'Fields to Match',
name: 'mergeByFields',
type: 'fixedCollection',
placeholder: 'Add Fields to Match',
default: { values: [{ field1: '', field2: '' }] },
typeOptions: {
multipleValues: true,
},
description: 'Specify the fields to use for matching input items',
displayOptions: {
show: {
advanced: [true],
},
},
options: [
{
displayName: 'Values',
name: 'values',
values: [
{
displayName: 'Input 1 Field',
name: 'field1',
type: 'string',
default: '',
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
placeholder: 'e.g. id',
hint: 'Drag or type the input field name',
requiresDataPath: 'single',
},
{
displayName: 'Input 2 Field',
name: 'field2',
type: 'string',
default: '',
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
placeholder: 'e.g. id',
hint: 'Drag or type the input field name',
requiresDataPath: 'single',
},
],
},
],
},
{
displayName: 'Output Type',
name: 'joinMode',
type: 'options',
description: 'How to select the items to send to output',
// eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items
options: [
{
name: 'Keep Matches',
value: 'keepMatches',
description: 'Items that match, merged together (inner join)',
},
{
name: 'Keep Non-Matches',
value: 'keepNonMatches',
description: "Items that don't match",
},
{
name: 'Keep Everything',
value: 'keepEverything',
description: "Items that match merged together, plus items that don't match (outer join)",
},
{
name: 'Enrich Input 1',
value: 'enrichInput1',
description: 'All of input 1, with data from input 2 added in (left join)',
},
{
name: 'Enrich Input 2',
value: 'enrichInput2',
description: 'All of input 2, with data from input 1 added in (right join)',
},
],
default: 'keepMatches',
},
{
displayName: 'Output Data From',
name: 'outputDataFrom',
type: 'options',
options: [
{
name: 'Both Inputs Merged Together',
value: 'both',
},
{
name: 'Input 1',
value: 'input1',
},
{
name: 'Input 2',
value: 'input2',
},
],
default: 'both',
displayOptions: {
show: {
joinMode: ['keepMatches'],
},
},
},
{
displayName: 'Output Data From',
name: 'outputDataFrom',
type: 'options',
options: [
{
name: 'Both Inputs Appended Together',
value: 'both',
},
{
name: 'Input 1',
value: 'input1',
},
{
name: 'Input 2',
value: 'input2',
},
],
default: 'both',
displayOptions: {
show: {
joinMode: ['keepNonMatches'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
...clashHandlingProperties,
displayOptions: {
hide: {
'/joinMode': ['keepMatches', 'keepNonMatches'],
},
},
},
{
...clashHandlingProperties,
displayOptions: {
show: {
'/joinMode': ['keepMatches'],
'/outputDataFrom': ['both'],
},
},
},
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
},
fuzzyCompareProperty,
{
...multipleMatchesProperty,
displayOptions: {
show: {
'/joinMode': ['keepMatches'],
'/outputDataFrom': ['both'],
},
},
},
{
...multipleMatchesProperty,
displayOptions: {
show: {
'/joinMode': ['enrichInput1', 'enrichInput2', 'keepEverything'],
},
},
},
],
},
];
const displayOptions = {
show: {
mode: ['combine'],
combineBy: ['combineByFields'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(
this: IExecuteFunctions,
inputsData: INodeExecutionData[][],
): Promise<INodeExecutionData[]> {
const returnData: INodeExecutionData[] = [];
const advanced = this.getNodeParameter('advanced', 0) as boolean;
let matchFields;
if (advanced) {
matchFields = this.getNodeParameter('mergeByFields.values', 0, []) as IDataObject[];
} else {
matchFields = (this.getNodeParameter('fieldsToMatchString', 0, '') as string)
.split(',')
.map((f) => {
const field = f.trim();
return { field1: field, field2: field };
});
}
matchFields = checkMatchFieldsInput(matchFields);
const joinMode = this.getNodeParameter('joinMode', 0) as MatchFieldsJoinMode;
const outputDataFrom = this.getNodeParameter('outputDataFrom', 0, 'both') as MatchFieldsOutput;
const options = this.getNodeParameter('options', 0, {}) as MatchFieldsOptions;
options.joinMode = joinMode;
options.outputDataFrom = outputDataFrom;
const nodeVersion = this.getNode().typeVersion;
let input1 = inputsData[0];
let input2 = inputsData[1];
if (nodeVersion < 2.1) {
input1 = checkInput(
this.getInputData(0),
matchFields.map((pair) => pair.field1),
options.disableDotNotation || false,
'Input 1',
);
if (!input1) return returnData;
input2 = checkInput(
this.getInputData(1),
matchFields.map((pair) => pair.field2),
options.disableDotNotation || false,
'Input 2',
);
} else {
if (!input1) return returnData;
}
if (input1.length === 0 || input2.length === 0) {
if (!input1.length && joinMode === 'keepNonMatches' && outputDataFrom === 'input1')
return returnData;
if (!input2.length && joinMode === 'keepNonMatches' && outputDataFrom === 'input2')
return returnData;
if (joinMode === 'keepMatches') {
// Stop the execution
return [];
} else if (joinMode === 'enrichInput1' && input1.length === 0) {
// No data to enrich so stop
return [];
} else if (joinMode === 'enrichInput2' && input2.length === 0) {
// No data to enrich so stop
return [];
} else {
// Return the data of any of the inputs that contains data
return [...input1, ...input2];
}
}
if (!input1) return returnData;
if (!input2 || !matchFields.length) {
if (
joinMode === 'keepMatches' ||
joinMode === 'keepEverything' ||
joinMode === 'enrichInput2'
) {
return returnData;
}
return input1;
}
const matches = findMatches(input1, input2, matchFields, options);
if (joinMode === 'keepMatches' || joinMode === 'keepEverything') {
let output: INodeExecutionData[] = [];
const clashResolveOptions = this.getNodeParameter(
'options.clashHandling.values',
0,
{},
) as ClashResolveOptions;
if (outputDataFrom === 'input1') {
output = matches.matched.map((match) => match.entry);
}
if (outputDataFrom === 'input2') {
output = matches.matched2;
}
if (outputDataFrom === 'both') {
output = mergeMatched(matches.matched, clashResolveOptions);
}
if (joinMode === 'keepEverything') {
let unmatched1 = matches.unmatched1;
let unmatched2 = matches.unmatched2;
if (clashResolveOptions.resolveClash === 'addSuffix') {
unmatched1 = addSuffixToEntriesKeys(unmatched1, '1');
unmatched2 = addSuffixToEntriesKeys(unmatched2, '2');
}
output = [...output, ...unmatched1, ...unmatched2];
}
returnData.push(...output);
}
if (joinMode === 'keepNonMatches') {
if (outputDataFrom === 'input1') {
return matches.unmatched1;
}
if (outputDataFrom === 'input2') {
return matches.unmatched2;
}
if (outputDataFrom === 'both') {
let output: INodeExecutionData[] = [];
output = output.concat(addSourceField(matches.unmatched1, 'input1'));
output = output.concat(addSourceField(matches.unmatched2, 'input2'));
return output;
}
}
if (joinMode === 'enrichInput1' || joinMode === 'enrichInput2') {
const clashResolveOptions = this.getNodeParameter(
'options.clashHandling.values',
0,
{},
) as ClashResolveOptions;
const mergedEntries = mergeMatched(matches.matched, clashResolveOptions, joinMode);
if (joinMode === 'enrichInput1') {
if (clashResolveOptions.resolveClash === 'addSuffix') {
returnData.push(...mergedEntries, ...addSuffixToEntriesKeys(matches.unmatched1, '1'));
} else {
returnData.push(...mergedEntries, ...matches.unmatched1);
}
} else {
if (clashResolveOptions.resolveClash === 'addSuffix') {
returnData.push(...mergedEntries, ...addSuffixToEntriesKeys(matches.unmatched2, '2'));
} else {
returnData.push(...mergedEntries, ...matches.unmatched2);
}
}
}
return returnData;
}

View File

@@ -0,0 +1,125 @@
import {
NodeExecutionOutput,
type IExecuteFunctions,
type INodeExecutionData,
type INodeProperties,
type IPairedItemData,
} from 'n8n-workflow';
import { updateDisplayOptions } from '@utils/utilities';
import type { ClashResolveOptions } from '../../helpers/interfaces';
import { clashHandlingProperties, numberInputsProperty } from '../../helpers/descriptions';
import { addSuffixToEntriesKeys, selectMergeMethod } from '../../helpers/utils';
import merge from 'lodash/merge';
export const properties: INodeProperties[] = [
numberInputsProperty,
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
...clashHandlingProperties,
default: { values: { resolveClash: 'addSuffix' } },
},
{
displayName: 'Include Any Unpaired Items',
name: 'includeUnpaired',
type: 'boolean',
default: false,
description:
'Whether unpaired items should be included in the result when there are differing numbers of items among the inputs',
},
],
},
];
const displayOptions = {
show: {
mode: ['combine'],
combineBy: ['combineByPosition'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(
this: IExecuteFunctions,
inputsData: INodeExecutionData[][],
): Promise<INodeExecutionData[]> {
const returnData: INodeExecutionData[] = [];
const clashHandling = this.getNodeParameter(
'options.clashHandling.values',
0,
{},
) as ClashResolveOptions;
const includeUnpaired = this.getNodeParameter('options.includeUnpaired', 0, false) as boolean;
let preferredInputIndex: number;
if (clashHandling?.resolveClash?.includes('preferInput')) {
preferredInputIndex = Number(clashHandling.resolveClash.replace('preferInput', '')) - 1;
} else {
preferredInputIndex = inputsData.length - 1;
}
const preferred = inputsData[preferredInputIndex];
if (clashHandling.resolveClash === 'addSuffix') {
for (const [inputIndex, input] of inputsData.entries()) {
inputsData[inputIndex] = addSuffixToEntriesKeys(input, String(inputIndex + 1));
}
}
let numEntries: number;
if (includeUnpaired) {
numEntries = Math.max(...inputsData.map((input) => input.length), preferred.length);
} else {
numEntries = Math.min(...inputsData.map((input) => input.length), preferred.length);
if (numEntries === 0) {
return new NodeExecutionOutput(
[returnData],
[
{
message:
'Consider enabling "Include Any Unpaired Items" in options or check your inputs',
},
],
);
}
}
const mergeIntoSingleObject = selectMergeMethod(clashHandling);
for (let i = 0; i < numEntries; i++) {
const preferredEntry = preferred[i] ?? {};
const restEntries = inputsData.map((input) => input[i] ?? {});
const json = {
...mergeIntoSingleObject(
{},
...restEntries.map((entry) => entry.json ?? {}),
preferredEntry.json ?? {},
),
};
const binary = {
...merge({}, ...restEntries.map((entry) => entry.binary ?? {}), preferredEntry.binary ?? {}),
};
const pairedItem = [
...restEntries.map((entry) => entry.pairedItem as IPairedItemData).flat(),
preferredEntry.pairedItem as IPairedItemData,
].filter((item) => item !== undefined);
returnData.push({ json, binary, pairedItem });
}
return returnData;
}

View File

@@ -0,0 +1,137 @@
import type {
IDataObject,
IExecuteFunctions,
INodeExecutionData,
INodeProperties,
IPairedItemData,
} from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import { getResolvables, updateDisplayOptions } from '@utils/utilities';
import { numberInputsProperty } from '../../helpers/descriptions';
import alasql from 'alasql';
import type { Database } from 'alasql';
export const properties: INodeProperties[] = [
numberInputsProperty,
{
displayName: 'Query',
name: 'query',
type: 'string',
default: 'SELECT * FROM input1 LEFT JOIN input2 ON input1.name = input2.id',
noDataExpression: true,
description: 'Input data available as tables with corresponding number, e.g. input1, input2',
hint: 'Supports <a href="https://github.com/alasql/alasql/wiki/Supported-SQL-statements" target="_blank">most</a> of the SQL-99 language',
required: true,
typeOptions: {
rows: 5,
editor: 'sqlEditor',
},
},
];
const displayOptions = {
show: {
mode: ['combineBySql'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(
this: IExecuteFunctions,
inputsData: INodeExecutionData[][],
): Promise<INodeExecutionData[]> {
const nodeId = this.getNode().id;
const returnData: INodeExecutionData[] = [];
const pairedItem: IPairedItemData[] = [];
const db: typeof Database = new (alasql as any).Database(nodeId);
try {
for (let i = 0; i < inputsData.length; i++) {
const inputData = inputsData[i];
inputData.forEach((item, index) => {
if (item.pairedItem === undefined) {
item.pairedItem = index;
}
if (typeof item.pairedItem === 'number') {
pairedItem.push({
item: item.pairedItem,
input: i,
});
return;
}
if (Array.isArray(item.pairedItem)) {
const pairedItems = item.pairedItem
.filter((p) => p !== undefined)
.map((p) => (typeof p === 'number' ? { item: p } : p))
.map((p) => {
return {
item: p.item,
input: i,
};
});
pairedItem.push(...pairedItems);
return;
}
pairedItem.push({
item: item.pairedItem.item,
input: i,
});
});
db.exec(`CREATE TABLE input${i + 1}`);
db.tables[`input${i + 1}`].data = inputData.map((entry) => entry.json);
}
} catch (error) {
throw new NodeOperationError(this.getNode(), error, {
message: 'Issue while creating table from',
description: error.message,
itemIndex: 0,
});
}
try {
let query = this.getNodeParameter('query', 0) as string;
for (const resolvable of getResolvables(query)) {
query = query.replace(resolvable, this.evaluateExpression(resolvable, 0) as string);
}
const result: IDataObject[] = db.exec(query);
for (const item of result) {
if (Array.isArray(item)) {
returnData.push(...item.map((json) => ({ json, pairedItem })));
} else if (typeof item === 'object') {
returnData.push({ json: item, pairedItem });
}
}
if (!returnData.length) {
returnData.push({ json: { success: true }, pairedItem });
}
} catch (error) {
let message = '';
if (typeof error === 'string') {
message = error;
} else {
message = error.message;
}
throw new NodeOperationError(this.getNode(), error, {
message: 'Issue while executing query',
description: message,
itemIndex: 0,
});
}
delete alasql.databases[nodeId];
return returnData;
}

View File

@@ -0,0 +1,77 @@
import type { INodeProperties } from 'n8n-workflow';
import * as append from './append';
import * as chooseBranch from './chooseBranch';
import * as combineAll from './combineAll';
import * as combineByFields from './combineByFields';
import * as combineBySql from './combineBySql';
import * as combineByPosition from './combineByPosition';
export { append, chooseBranch, combineAll, combineByFields, combineBySql, combineByPosition };
export const description: INodeProperties[] = [
{
displayName: 'Mode',
name: 'mode',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Append',
value: 'append',
description: 'Output items of each input, one after the other',
},
{
name: 'Combine',
value: 'combine',
description: 'Merge matching items together',
},
{
name: 'SQL Query',
value: 'combineBySql',
description: 'Write a query to do the merge',
},
{
name: 'Choose Branch',
value: 'chooseBranch',
description: 'Output data from a specific branch, without modifying it',
},
],
default: 'append',
description: 'How input data should be merged',
},
{
displayName: 'Combine By',
name: 'combineBy',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Matching Fields',
value: 'combineByFields',
description: 'Combine items with the same field values',
},
{
name: 'Position',
value: 'combineByPosition',
description: 'Combine items based on their order',
},
{
name: 'All Possible Combinations',
value: 'combineAll',
description: 'Every pairing of every two items (cross join)',
},
],
default: 'combineByFields',
description: 'How input data should be merged',
displayOptions: {
show: { mode: ['combine'] },
},
},
...append.description,
...combineAll.description,
...combineByFields.description,
...combineBySql.description,
...combineByPosition.description,
...chooseBranch.description,
];

View File

@@ -0,0 +1,7 @@
export type MergeType =
| 'append'
| 'combineByFields'
| 'combineBySql'
| 'combineByPosition'
| 'combineAll'
| 'chooseBranch';

View File

@@ -0,0 +1,22 @@
import { NodeExecutionOutput, type IExecuteFunctions } from 'n8n-workflow';
import type { MergeType } from './node.type';
import * as mode from './mode';
import { getNodeInputsData } from '../helpers/utils';
export async function router(this: IExecuteFunctions) {
const inputsData = getNodeInputsData.call(this);
let operationMode = this.getNodeParameter('mode', 0) as string;
if (operationMode === 'combine') {
const combineBy = this.getNodeParameter('combineBy', 0) as string;
operationMode = combineBy;
}
const returnData = await mode[operationMode as MergeType].execute.call(this, inputsData);
if (returnData instanceof NodeExecutionOutput) {
return returnData;
} else {
return [returnData];
}
}

View File

@@ -0,0 +1,23 @@
/* eslint-disable n8n-nodes-base/node-filename-against-convention */
import type { INodeTypeDescription } from 'n8n-workflow';
import * as mode from './mode';
import { configuredInputs } from '../helpers/utils';
export const versionDescription: INodeTypeDescription = {
displayName: 'Merge',
name: 'merge',
group: ['transform'],
description: 'Merges data of multiple streams once data from both is available',
version: [3],
defaults: {
name: 'Merge',
},
inputs: `={{(${configuredInputs})($parameter)}}`,
outputs: ['main'],
// If mode is chooseBranch data from both branches is required
// to continue, else data from any input suffices
requiredInputs: '={{ $parameter["mode"] === "chooseBranch" ? [0, 1] : 1 }}',
properties: [...mode.description],
};

View File

@@ -0,0 +1,125 @@
import type { INodeProperties } from 'n8n-workflow';
export const fuzzyCompareProperty: INodeProperties = {
displayName: 'Fuzzy Compare',
name: 'fuzzyCompare',
type: 'boolean',
default: false,
description:
"Whether to tolerate small type differences when comparing fields. E.g. the number 3 and the string '3' are treated as the same.",
};
export const numberInputsProperty: INodeProperties = {
displayName: 'Number of Inputs',
name: 'numberInputs',
type: 'options',
noDataExpression: true,
default: 2,
options: [
{
name: '2',
value: 2,
},
{
name: '3',
value: 3,
},
{
name: '4',
value: 4,
},
{
name: '5',
value: 5,
},
{
name: '6',
value: 6,
},
{
name: '7',
value: 7,
},
{
name: '8',
value: 8,
},
{
name: '9',
value: 9,
},
{
name: '10',
value: 10,
},
],
validateType: 'number',
description:
'The number of data inputs you want to merge. The node waits for all connected inputs to be executed.',
};
export const clashHandlingProperties: INodeProperties = {
displayName: 'Clash Handling',
name: 'clashHandling',
type: 'fixedCollection',
default: {
values: { resolveClash: 'preferLast', mergeMode: 'deepMerge', overrideEmpty: false },
},
options: [
{
displayName: 'Values',
name: 'values',
values: [
{
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options
displayName: 'When Field Values Clash',
name: 'resolveClash',
// eslint-disable-next-line n8n-nodes-base/node-param-description-missing-from-dynamic-options
type: 'options',
default: '',
typeOptions: {
loadOptionsMethod: 'getResolveClashOptions',
loadOptionsDependsOn: ['numberInputs'],
},
},
{
displayName: 'Merging Nested Fields',
name: 'mergeMode',
type: 'options',
default: 'deepMerge',
options: [
{
name: 'Deep Merge',
value: 'deepMerge',
description: 'Merge at every level of nesting',
},
{
name: 'Shallow Merge',
value: 'shallowMerge',
description:
'Merge at the top level only (all nested fields will come from the same input)',
},
],
hint: 'How to merge when there are sub-fields below the top-level ones',
displayOptions: {
show: {
resolveClash: [{ _cnd: { not: 'addSuffix' } }],
},
},
},
{
displayName: 'Minimize Empty Fields',
name: 'overrideEmpty',
type: 'boolean',
default: false,
description:
"Whether to override the preferred input version for a field if it is empty and the other version isn't. Here 'empty' means undefined, null or an empty string.",
displayOptions: {
show: {
resolveClash: [{ _cnd: { not: 'addSuffix' } }],
},
},
},
],
},
],
};

View File

@@ -0,0 +1,27 @@
type MultipleMatches = 'all' | 'first';
export type MatchFieldsOptions = {
joinMode: MatchFieldsJoinMode;
outputDataFrom: MatchFieldsOutput;
multipleMatches: MultipleMatches;
disableDotNotation: boolean;
fuzzyCompare?: boolean;
};
type ClashMergeMode = 'deepMerge' | 'shallowMerge';
type ClashResolveMode = 'addSuffix' | 'preferInput1' | 'preferLast';
export type ClashResolveOptions = {
resolveClash: ClashResolveMode;
mergeMode: ClashMergeMode;
overrideEmpty: boolean;
};
export type MatchFieldsOutput = 'both' | 'input1' | 'input2';
export type MatchFieldsJoinMode =
| 'keepEverything'
| 'keepMatches'
| 'keepNonMatches'
| 'enrichInput2'
| 'enrichInput1';

View File

@@ -0,0 +1,389 @@
import { ApplicationError, NodeConnectionType, NodeHelpers } from 'n8n-workflow';
import type {
GenericValue,
IBinaryKeyData,
IDataObject,
IExecuteFunctions,
INodeExecutionData,
INodeParameters,
IPairedItemData,
} from 'n8n-workflow';
import assign from 'lodash/assign';
import assignWith from 'lodash/assignWith';
import get from 'lodash/get';
import merge from 'lodash/merge';
import mergeWith from 'lodash/mergeWith';
import { fuzzyCompare, preparePairedItemDataArray } from '@utils/utilities';
import type { ClashResolveOptions, MatchFieldsJoinMode, MatchFieldsOptions } from './interfaces';
type PairToMatch = {
field1: string;
field2: string;
};
type EntryMatches = {
entry: INodeExecutionData;
matches: INodeExecutionData[];
};
type CompareFunction = <T, U>(a: T, b: U) => boolean;
export function addSuffixToEntriesKeys(data: INodeExecutionData[], suffix: string) {
return data.map((entry) => {
const json: IDataObject = {};
Object.keys(entry.json).forEach((key) => {
json[`${key}_${suffix}`] = entry.json[key];
});
return { ...entry, json };
});
}
function findAllMatches(
data: INodeExecutionData[],
lookup: IDataObject,
disableDotNotation: boolean,
isEntriesEqual: CompareFunction,
) {
return data.reduce((acc, entry2, i) => {
if (entry2 === undefined) return acc;
for (const key of Object.keys(lookup)) {
const expectedValue = lookup[key];
let entry2FieldValue;
if (disableDotNotation) {
entry2FieldValue = entry2.json[key];
} else {
entry2FieldValue = get(entry2.json, key);
}
if (!isEntriesEqual(expectedValue, entry2FieldValue)) {
return acc;
}
}
return acc.concat({
entry: entry2,
index: i,
});
}, [] as IDataObject[]);
}
function findFirstMatch(
data: INodeExecutionData[],
lookup: IDataObject,
disableDotNotation: boolean,
isEntriesEqual: CompareFunction,
) {
const index = data.findIndex((entry2) => {
if (entry2 === undefined) return false;
for (const key of Object.keys(lookup)) {
const expectedValue = lookup[key];
let entry2FieldValue;
if (disableDotNotation) {
entry2FieldValue = entry2.json[key];
} else {
entry2FieldValue = get(entry2.json, key);
}
if (!isEntriesEqual(expectedValue, entry2FieldValue)) {
return false;
}
}
return true;
});
if (index === -1) return [];
return [{ entry: data[index], index }];
}
export function findMatches(
input1: INodeExecutionData[],
input2: INodeExecutionData[],
fieldsToMatch: PairToMatch[],
options: MatchFieldsOptions,
) {
const data1 = [...input1];
const data2 = [...input2];
const isEntriesEqual = fuzzyCompare(options.fuzzyCompare as boolean);
const disableDotNotation = options.disableDotNotation || false;
const multipleMatches = (options.multipleMatches as string) || 'all';
const filteredData = {
matched: [] as EntryMatches[],
matched2: [] as INodeExecutionData[],
unmatched1: [] as INodeExecutionData[],
unmatched2: [] as INodeExecutionData[],
};
const matchedInInput2 = new Set<number>();
matchesLoop: for (const entry1 of data1) {
const lookup: IDataObject = {};
fieldsToMatch.forEach((matchCase) => {
let valueToCompare;
if (disableDotNotation) {
valueToCompare = entry1.json[matchCase.field1];
} else {
valueToCompare = get(entry1.json, matchCase.field1);
}
lookup[matchCase.field2] = valueToCompare;
});
for (const fieldValue of Object.values(lookup)) {
if (fieldValue === undefined) {
filteredData.unmatched1.push(entry1);
continue matchesLoop;
}
}
const foundedMatches =
multipleMatches === 'all'
? findAllMatches(data2, lookup, disableDotNotation, isEntriesEqual)
: findFirstMatch(data2, lookup, disableDotNotation, isEntriesEqual);
const matches = foundedMatches.map((match) => match.entry) as INodeExecutionData[];
foundedMatches.map((match) => matchedInInput2.add(match.index as number));
if (matches.length) {
if (
options.outputDataFrom === 'both' ||
options.joinMode === 'enrichInput1' ||
options.joinMode === 'enrichInput2'
) {
matches.forEach((match) => {
filteredData.matched.push({
entry: entry1,
matches: [match],
});
});
} else {
filteredData.matched.push({
entry: entry1,
matches,
});
}
} else {
filteredData.unmatched1.push(entry1);
}
}
data2.forEach((entry, i) => {
if (matchedInInput2.has(i)) {
filteredData.matched2.push(entry);
} else {
filteredData.unmatched2.push(entry);
}
});
return filteredData;
}
export function selectMergeMethod(clashResolveOptions: ClashResolveOptions) {
const mergeMode = clashResolveOptions.mergeMode as string;
if (clashResolveOptions.overrideEmpty) {
function customizer(targetValue: GenericValue, srcValue: GenericValue) {
if (srcValue === undefined || srcValue === null || srcValue === '') {
return targetValue;
}
}
if (mergeMode === 'deepMerge') {
return (target: IDataObject, ...source: IDataObject[]) => {
const targetCopy = Object.assign({}, target);
return mergeWith(targetCopy, ...source, customizer);
};
}
if (mergeMode === 'shallowMerge') {
return (target: IDataObject, ...source: IDataObject[]) => {
const targetCopy = Object.assign({}, target);
return assignWith(targetCopy, ...source, customizer);
};
}
} else {
if (mergeMode === 'deepMerge') {
return (target: IDataObject, ...source: IDataObject[]) => merge({}, target, ...source);
}
if (mergeMode === 'shallowMerge') {
return (target: IDataObject, ...source: IDataObject[]) => assign({}, target, ...source);
}
}
return (target: IDataObject, ...source: IDataObject[]) => merge({}, target, ...source);
}
export function mergeMatched(
matched: EntryMatches[],
clashResolveOptions: ClashResolveOptions,
joinMode?: MatchFieldsJoinMode,
) {
const returnData: INodeExecutionData[] = [];
let resolveClash = clashResolveOptions.resolveClash as string;
const mergeIntoSingleObject = selectMergeMethod(clashResolveOptions);
for (const match of matched) {
let { entry, matches } = match;
let json: IDataObject = {};
let binary: IBinaryKeyData = {};
let pairedItem: IPairedItemData[] = [];
if (resolveClash === 'addSuffix') {
const suffix1 = '1';
const suffix2 = '2';
[entry] = addSuffixToEntriesKeys([entry], suffix1);
matches = addSuffixToEntriesKeys(matches, suffix2);
json = mergeIntoSingleObject({ ...entry.json }, ...matches.map((item) => item.json));
binary = mergeIntoSingleObject(
{ ...entry.binary },
...matches.map((item) => item.binary as IDataObject),
);
pairedItem = [
...preparePairedItemDataArray(entry.pairedItem),
...matches.map((item) => preparePairedItemDataArray(item.pairedItem)).flat(),
];
} else {
const preferInput1 = 'preferInput1';
const preferLast = 'preferLast';
if (resolveClash === undefined) {
if (joinMode !== 'enrichInput2') {
resolveClash = 'preferLast';
} else {
resolveClash = 'preferInput1';
}
}
if (resolveClash === preferInput1) {
const [firstMatch, ...restMatches] = matches;
json = mergeIntoSingleObject(
{ ...firstMatch.json },
...restMatches.map((item) => item.json),
entry.json,
);
binary = mergeIntoSingleObject(
{ ...firstMatch.binary },
...restMatches.map((item) => item.binary as IDataObject),
entry.binary as IDataObject,
);
pairedItem = [
...preparePairedItemDataArray(firstMatch.pairedItem),
...restMatches.map((item) => preparePairedItemDataArray(item.pairedItem)).flat(),
...preparePairedItemDataArray(entry.pairedItem),
];
}
if (resolveClash === preferLast) {
json = mergeIntoSingleObject({ ...entry.json }, ...matches.map((item) => item.json));
binary = mergeIntoSingleObject(
{ ...entry.binary },
...matches.map((item) => item.binary as IDataObject),
);
pairedItem = [
...preparePairedItemDataArray(entry.pairedItem),
...matches.map((item) => preparePairedItemDataArray(item.pairedItem)).flat(),
];
}
}
returnData.push({
json,
binary,
pairedItem,
});
}
return returnData;
}
export function checkMatchFieldsInput(data: IDataObject[]) {
if (data.length === 1 && data[0].field1 === '' && data[0].field2 === '') {
throw new ApplicationError(
'You need to define at least one pair of fields in "Fields to Match" to match on',
{ level: 'warning' },
);
}
for (const [index, pair] of data.entries()) {
if (pair.field1 === '' || pair.field2 === '') {
throw new ApplicationError(
`You need to define both fields in "Fields to Match" for pair ${index + 1},
field 1 = '${pair.field1}'
field 2 = '${pair.field2}'`,
{ level: 'warning' },
);
}
}
return data as PairToMatch[];
}
export function checkInput(
input: INodeExecutionData[],
fields: string[],
disableDotNotation: boolean,
inputLabel: string,
) {
for (const field of fields) {
const isPresent = (input || []).some((entry) => {
if (disableDotNotation) {
return entry.json.hasOwnProperty(field);
}
return get(entry.json, field, undefined) !== undefined;
});
if (!isPresent) {
throw new ApplicationError(
`Field '${field}' is not present in any of items in '${inputLabel}'`,
{ level: 'warning' },
);
}
}
return input;
}
export function addSourceField(data: INodeExecutionData[], sourceField: string) {
return data.map((entry) => {
const json = {
...entry.json,
_source: sourceField,
};
return {
...entry,
json,
};
});
}
export const configuredInputs = (parameters: INodeParameters) => {
return Array.from({ length: (parameters.numberInputs as number) || 2 }, (_, i) => ({
type: `${NodeConnectionType.Main}`,
displayName: `Input ${(i + 1).toString()}`,
}));
};
export function getNodeInputsData(this: IExecuteFunctions) {
const returnData: INodeExecutionData[][] = [];
const inputs = NodeHelpers.getConnectionTypes(this.getNodeInputs()).filter(
(type) => type === NodeConnectionType.Main,
);
for (let i = 0; i < inputs.length; i++) {
try {
returnData.push(this.getInputData(i) ?? []);
} catch (error) {
returnData.push([]);
}
}
return returnData;
}

View File

@@ -0,0 +1 @@
export * as loadOptions from './loadOptions';

View File

@@ -0,0 +1,49 @@
import type { ILoadOptionsFunctions, INodePropertyOptions } from 'n8n-workflow';
export async function getResolveClashOptions(
this: ILoadOptionsFunctions,
): Promise<INodePropertyOptions[]> {
const numberOfInputs = this.getNodeParameter('numberInputs', 2) as number;
if (numberOfInputs <= 2) {
return [
{
name: 'Always Add Input Number to Field Names',
value: 'addSuffix',
},
{
name: 'Prefer Input 1 Version',
value: 'preferInput1',
},
{
name: 'Prefer Input 2 Version',
value: 'preferLast',
},
];
} else {
return [
{
name: 'Always Add Input Number to Field Names',
value: 'addSuffix',
},
{
name: 'Use Earliest Version',
value: 'preferInput1',
},
];
}
}
export async function getInputs(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
const numberOfInputs = this.getNodeParameter('numberInputs', 2) as number;
const returnData: INodePropertyOptions[] = [];
for (let i = 0; i < numberOfInputs; i++) {
returnData.push({
name: `${i + 1}`,
value: i + 1,
});
}
return returnData;
}