feat(Google BigQuery Node): Node improvements (#4877)
* ⚡ setup * ⚡ finished v2 setup * ⚡ fix return all, fix simplify with nested schema * ⚡ fix for external tables, updated scopes * ⚡ query operation * ⚡ linter fixes * ⚡ fixed not processed errors when inserting, move main loop to execute function to allow bulk request * ⚡ customizible batch size when inserting, improoved errors * ⚡ options for mapping input * ⚡ fix for inserting RECORD type * ⚡ updated simplify logic * ⚡ fix for return with selected fields * ⚡ option to return table schema * ⚡ linter fixes * ⚡ fix imports * ⚡ query resource and fixes, rlc for projects * ⚡ removed simplify, added raw output option * ⚡ rlc for tables and datasets, no urls option * ⚡ updated hints and description of query parameter, fix getMany VIEW, multioptions fo fields * ⚡ added case when rows are empty * ⚡ linter fixes * ⚡ UI update, one resource * ⚡ fix for output with field named json * ⚡ using jobs instead queries * ⚡ added error message * ⚡ search for RLCs, fixes * ⚡ json processing * ⚡ removed getAll operation * ⚡ executeQuery update * ⚡ unit test * ⚡ tests setup, fixes * ⚡ tests * Remove script for checking unused loadOptions --------- Co-authored-by: agobrech <ael.gobrecht@gmail.com>
This commit is contained in:
Michael Kret
GitHub
@@ -0,0 +1,79 @@
|
||||
import type { INodeTypes } from 'n8n-workflow';
|
||||
|
||||
import { setup, workflowToTests } from '../../../../../../test/nodes/Helpers';
|
||||
import type { WorkflowTestData } from '../../../../../../test/nodes/types';
|
||||
import { executeWorkflow } from '../../../../../../test/nodes/ExecuteWorkflow';
|
||||
|
||||
import * as transport from '../../../v2/transport';
|
||||
|
||||
import nock from 'nock';
|
||||
|
||||
jest.mock('../../../v2/transport', () => {
|
||||
const originalModule = jest.requireActual('../../../v2/transport');
|
||||
return {
|
||||
...originalModule,
|
||||
googleApiRequest: jest.fn(async (method: string, resource: string) => {
|
||||
if (resource === '/v2/projects/test-project/jobs' && method === 'POST') {
|
||||
return Promise.resolve({
|
||||
jobReference: {
|
||||
jobId: 'job_123',
|
||||
},
|
||||
status: {
|
||||
state: 'DONE',
|
||||
},
|
||||
});
|
||||
}
|
||||
if (resource === '/v2/projects/test-project/queries/job_123' && method === 'GET') {
|
||||
return Promise.resolve({});
|
||||
}
|
||||
return Promise.resolve();
|
||||
}),
|
||||
// googleApiRequestAllItems: jest.fn(async () => Promise.resolve()),
|
||||
};
|
||||
});
|
||||
|
||||
describe('Test Google BigQuery V2, executeQuery', () => {
|
||||
const workflows = ['nodes/Google/BigQuery/test/v2/node/executeQuery.workflow.json'];
|
||||
const tests = workflowToTests(workflows);
|
||||
|
||||
beforeAll(() => {
|
||||
nock.disableNetConnect();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
nock.restore();
|
||||
jest.unmock('../../../v2/transport');
|
||||
});
|
||||
|
||||
const nodeTypes = setup(tests);
|
||||
|
||||
const testNode = async (testData: WorkflowTestData, types: INodeTypes) => {
|
||||
const { result } = await executeWorkflow(testData, types);
|
||||
|
||||
expect(transport.googleApiRequest).toHaveBeenCalledTimes(2);
|
||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
||||
'POST',
|
||||
'/v2/projects/test-project/jobs',
|
||||
{
|
||||
configuration: {
|
||||
query: {
|
||||
query: 'SELECT * FROM bigquery_node_dev_test_dataset.test_json;',
|
||||
useLegacySql: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
||||
'GET',
|
||||
'/v2/projects/test-project/queries/job_123',
|
||||
undefined,
|
||||
{},
|
||||
);
|
||||
|
||||
expect(result.finished).toEqual(true);
|
||||
};
|
||||
|
||||
for (const testData of tests) {
|
||||
test(testData.description, async () => testNode(testData, nodeTypes));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"name": "My workflow 12",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "7db7d51a-83c2-4aa0-a736-9c3d1c031b60",
|
||||
"name": "When clicking \"Execute Workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [360, 340]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"authentication": "serviceAccount",
|
||||
"projectId": {
|
||||
"__rl": true,
|
||||
"value": "test-project",
|
||||
"mode": "list",
|
||||
"cachedResultName": "test-project",
|
||||
"cachedResultUrl": "https://console.cloud.google.com/bigquery?project=test-project"
|
||||
},
|
||||
"sqlQuery": "SELECT * FROM bigquery_node_dev_test_dataset.test_json;",
|
||||
"options": {}
|
||||
},
|
||||
"id": "83d00275-0f98-4d5e-a3d6-bbca940ff8ac",
|
||||
"name": "Google BigQuery",
|
||||
"type": "n8n-nodes-base.googleBigQuery",
|
||||
"typeVersion": 2,
|
||||
"position": [620, 340],
|
||||
"credentials": {
|
||||
"googleApi": {
|
||||
"id": "66",
|
||||
"name": "Google account 5"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"Google BigQuery": []
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute Workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Google BigQuery",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {},
|
||||
"versionId": "be2fc126-5d71-4e86-9a4e-eb62ad266860",
|
||||
"id": "156",
|
||||
"meta": {
|
||||
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
|
||||
},
|
||||
"tags": []
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import type { INodeTypes } from 'n8n-workflow';
|
||||
|
||||
import { setup, workflowToTests } from '../../../../../../test/nodes/Helpers';
|
||||
import type { WorkflowTestData } from '../../../../../../test/nodes/types';
|
||||
import { executeWorkflow } from '../../../../../../test/nodes/ExecuteWorkflow';
|
||||
import nock from 'nock';
|
||||
|
||||
import * as transport from '../../../v2/transport';
|
||||
|
||||
jest.mock('../../../v2/transport', () => {
|
||||
const originalModule = jest.requireActual('../../../v2/transport');
|
||||
return {
|
||||
...originalModule,
|
||||
googleApiRequest: jest.fn(async (method: string, resource: string) => {
|
||||
if (
|
||||
resource ===
|
||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text' &&
|
||||
method === 'GET'
|
||||
) {
|
||||
return Promise.resolve({
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: 'id', type: 'INT' },
|
||||
{ name: 'test', type: 'STRING' },
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
if (
|
||||
resource ===
|
||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text/insertAll' &&
|
||||
method === 'POST'
|
||||
) {
|
||||
return Promise.resolve({ kind: 'bigquery#tableDataInsertAllResponse' });
|
||||
}
|
||||
return Promise.resolve();
|
||||
}),
|
||||
googleApiRequestAllItems: jest.fn(async () => Promise.resolve()),
|
||||
};
|
||||
});
|
||||
|
||||
describe('Test Google BigQuery V2, insert auto map', () => {
|
||||
const workflows = ['nodes/Google/BigQuery/test/v2/node/insert.autoMapMode.workflow.json'];
|
||||
const tests = workflowToTests(workflows);
|
||||
|
||||
beforeAll(() => {
|
||||
nock.disableNetConnect();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
nock.restore();
|
||||
jest.unmock('../../../v2/transport');
|
||||
});
|
||||
|
||||
const nodeTypes = setup(tests);
|
||||
|
||||
const testNode = async (testData: WorkflowTestData, types: INodeTypes) => {
|
||||
const { result } = await executeWorkflow(testData, types);
|
||||
|
||||
expect(transport.googleApiRequest).toHaveBeenCalledTimes(2);
|
||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
||||
'GET',
|
||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text',
|
||||
{},
|
||||
);
|
||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
||||
'POST',
|
||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text/insertAll',
|
||||
{
|
||||
rows: [
|
||||
{ json: { id: 1, test: '111' } },
|
||||
{ json: { id: 2, test: '222' } },
|
||||
{ json: { id: 3, test: '333' } },
|
||||
],
|
||||
traceId: 'trace_id',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.finished).toEqual(true);
|
||||
};
|
||||
|
||||
for (const testData of tests) {
|
||||
test(testData.description, async () => testNode(testData, nodeTypes));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
{
|
||||
"name": "My workflow 12",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "7db7d51a-83c2-4aa0-a736-9c3d1c031b60",
|
||||
"name": "When clicking \"Execute Workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [20, 340]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"authentication": "serviceAccount",
|
||||
"operation": "insert",
|
||||
"projectId": {
|
||||
"__rl": true,
|
||||
"value": "test-project",
|
||||
"mode": "list",
|
||||
"cachedResultName": "test-project",
|
||||
"cachedResultUrl": "https://console.cloud.google.com/bigquery?project=test-project"
|
||||
},
|
||||
"datasetId": {
|
||||
"__rl": true,
|
||||
"value": "bigquery_node_dev_test_dataset",
|
||||
"mode": "list",
|
||||
"cachedResultName": "bigquery_node_dev_test_dataset"
|
||||
},
|
||||
"tableId": {
|
||||
"__rl": true,
|
||||
"value": "num_text",
|
||||
"mode": "list",
|
||||
"cachedResultName": "num_text"
|
||||
},
|
||||
"options": {
|
||||
"traceId": "trace_id"
|
||||
}
|
||||
},
|
||||
"id": "83d00275-0f98-4d5e-a3d6-bbca940ff8ac",
|
||||
"name": "Google BigQuery",
|
||||
"type": "n8n-nodes-base.googleBigQuery",
|
||||
"typeVersion": 2,
|
||||
"position": [500, 340],
|
||||
"credentials": {
|
||||
"googleApi": {
|
||||
"id": "66",
|
||||
"name": "Google account 5"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"jsCode": "return [\n{\n\"id\":\n1,\n\"test\":\n\"111\"\n},\n{\n\"id\":\n2,\n\"test\":\n\"222\"\n},\n{\n\"id\":\n3,\n\"test\":\n\"333\"\n},\n];"
|
||||
},
|
||||
"id": "11d06660-cbd3-4bd2-9619-68e82438a0e3",
|
||||
"name": "Code",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 1,
|
||||
"position": [240, 340]
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"Code": [
|
||||
{
|
||||
"json": {
|
||||
"id": 1,
|
||||
"test": "111"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 2,
|
||||
"test": "222"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"id": 3,
|
||||
"test": "333"
|
||||
}
|
||||
}
|
||||
],
|
||||
"Google BigQuery": [
|
||||
{
|
||||
"json": {
|
||||
"kind": "bigquery#tableDataInsertAllResponse"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"kind": "bigquery#tableDataInsertAllResponse"
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"kind": "bigquery#tableDataInsertAllResponse"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute Workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Code",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Code": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Google BigQuery",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {},
|
||||
"versionId": "30d3e38a-b5a4-4999-816d-7c05a68f31c8",
|
||||
"id": "156",
|
||||
"meta": {
|
||||
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
|
||||
},
|
||||
"tags": []
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import type { INodeTypes } from 'n8n-workflow';
|
||||
|
||||
import { setup, workflowToTests } from '../../../../../../test/nodes/Helpers';
|
||||
import type { WorkflowTestData } from '../../../../../../test/nodes/types';
|
||||
import { executeWorkflow } from '../../../../../../test/nodes/ExecuteWorkflow';
|
||||
import nock from 'nock';
|
||||
|
||||
import * as transport from '../../../v2/transport';
|
||||
|
||||
jest.mock('../../../v2/transport', () => {
|
||||
const originalModule = jest.requireActual('../../../v2/transport');
|
||||
return {
|
||||
...originalModule,
|
||||
googleApiRequest: jest.fn(async (method: string, resource: string) => {
|
||||
if (
|
||||
resource ===
|
||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json' &&
|
||||
method === 'GET'
|
||||
) {
|
||||
return Promise.resolve({
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: 'json', type: 'JSON' },
|
||||
{ name: 'name with space', type: 'STRING' },
|
||||
{ name: 'active', type: 'BOOLEAN' },
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
if (
|
||||
resource ===
|
||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json/insertAll' &&
|
||||
method === 'POST'
|
||||
) {
|
||||
return Promise.resolve({ kind: 'bigquery#tableDataInsertAllResponse' });
|
||||
}
|
||||
return Promise.resolve();
|
||||
}),
|
||||
googleApiRequestAllItems: jest.fn(async () => Promise.resolve()),
|
||||
};
|
||||
});
|
||||
|
||||
describe('Test Google BigQuery V2, insert define manualy', () => {
|
||||
const workflows = ['nodes/Google/BigQuery/test/v2/node/insert.manualMode.workflow.json'];
|
||||
const tests = workflowToTests(workflows);
|
||||
|
||||
beforeAll(() => {
|
||||
nock.disableNetConnect();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
nock.restore();
|
||||
jest.unmock('../../../v2/transport');
|
||||
});
|
||||
|
||||
const nodeTypes = setup(tests);
|
||||
|
||||
const testNode = async (testData: WorkflowTestData, types: INodeTypes) => {
|
||||
const { result } = await executeWorkflow(testData, types);
|
||||
|
||||
expect(transport.googleApiRequest).toHaveBeenCalledTimes(2);
|
||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
||||
'GET',
|
||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json',
|
||||
{},
|
||||
);
|
||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
||||
'POST',
|
||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json/insertAll',
|
||||
{
|
||||
rows: [{ json: { active: 'true', json: '{"test": 1}', 'name with space': 'some name' } }],
|
||||
traceId: 'trace_id',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.finished).toEqual(true);
|
||||
};
|
||||
|
||||
for (const testData of tests) {
|
||||
test(testData.description, async () => testNode(testData, nodeTypes));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
{
|
||||
"name": "My workflow 12",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {},
|
||||
"id": "7db7d51a-83c2-4aa0-a736-9c3d1c031b60",
|
||||
"name": "When clicking \"Execute Workflow\"",
|
||||
"type": "n8n-nodes-base.manualTrigger",
|
||||
"typeVersion": 1,
|
||||
"position": [360, 340]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"authentication": "serviceAccount",
|
||||
"operation": "insert",
|
||||
"projectId": {
|
||||
"__rl": true,
|
||||
"value": "test-project",
|
||||
"mode": "list",
|
||||
"cachedResultName": "test-project",
|
||||
"cachedResultUrl": "https://console.cloud.google.com/bigquery?project=test-project"
|
||||
},
|
||||
"datasetId": {
|
||||
"__rl": true,
|
||||
"value": "bigquery_node_dev_test_dataset",
|
||||
"mode": "list",
|
||||
"cachedResultName": "bigquery_node_dev_test_dataset"
|
||||
},
|
||||
"tableId": {
|
||||
"__rl": true,
|
||||
"value": "test_json",
|
||||
"mode": "list",
|
||||
"cachedResultName": "test_json"
|
||||
},
|
||||
"dataMode": "define",
|
||||
"fieldsUi": {
|
||||
"values": [
|
||||
{
|
||||
"fieldId": "active",
|
||||
"fieldValue": "true"
|
||||
},
|
||||
{
|
||||
"fieldId": "name with space",
|
||||
"fieldValue": "some name"
|
||||
},
|
||||
{
|
||||
"fieldId": "json",
|
||||
"fieldValue": "{\"test\": 1}"
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"traceId": "trace_id"
|
||||
}
|
||||
},
|
||||
"id": "83d00275-0f98-4d5e-a3d6-bbca940ff8ac",
|
||||
"name": "Google BigQuery",
|
||||
"type": "n8n-nodes-base.googleBigQuery",
|
||||
"typeVersion": 2,
|
||||
"position": [620, 340],
|
||||
"credentials": {
|
||||
"googleApi": {
|
||||
"id": "66",
|
||||
"name": "Google account 5"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"pinData": {
|
||||
"Google BigQuery": [
|
||||
{
|
||||
"json": {
|
||||
"kind": "bigquery#tableDataInsertAllResponse"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"connections": {
|
||||
"When clicking \"Execute Workflow\"": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Google BigQuery",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"active": false,
|
||||
"settings": {},
|
||||
"versionId": "abd49f26-184d-4f9b-95f0-389ea20df809",
|
||||
"id": "156",
|
||||
"meta": {
|
||||
"instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0"
|
||||
},
|
||||
"tags": []
|
||||
}
|
||||
Reference in New Issue
Block a user