feat(Google BigQuery Node): Node improvements (#4877)

* ⚡ setup

* ⚡ finished v2 setup

* ⚡ fix return all, fix simplify with nested schema

* ⚡ fix for external tables, updated scopes

* ⚡ query operation

* ⚡ linter fixes

* ⚡ fixed not processed errors when inserting, move main loop to execute function to allow bulk request

* ⚡ customizible batch size when inserting, improoved errors

* ⚡ options for mapping input

* ⚡ fix for inserting RECORD type

* ⚡ updated simplify logic

* ⚡ fix for return with  selected fields

* ⚡ option to return table schema

* ⚡ linter fixes

* ⚡ fix imports

* ⚡ query resource and fixes, rlc for projects

* ⚡ removed simplify, added raw output option

* ⚡ rlc for tables and datasets, no urls option

* ⚡ updated hints and description of query parameter, fix getMany VIEW, multioptions fo fields

* ⚡ added case when rows are empty

* ⚡ linter fixes

* ⚡ UI update, one resource

* ⚡ fix for output with field named json

* ⚡ using jobs instead queries

* ⚡ added error message

* ⚡ search for RLCs, fixes

* ⚡ json processing

* ⚡ removed getAll operation

* ⚡ executeQuery update

* ⚡ unit test

* ⚡ tests setup, fixes

* ⚡ tests

* Remove script for checking unused loadOptions

---------

Co-authored-by: agobrech <ael.gobrecht@gmail.com>

packages/nodes-base/nodes/Google/BigQuery/v2/methods/index.ts

@@ -0,0 +1,2 @@
+export * as loadOptions from './loadOptions';
+export * as listSearch from './listSearch';

packages/nodes-base/nodes/Google/BigQuery/v2/methods/listSearch.ts

@@ -0,0 +1,115 @@
+import type { IDataObject, ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow';
+import { googleApiRequest } from '../transport';
+
+export async function searchProjects(
+	this: ILoadOptionsFunctions,
+	filter?: string,
+	paginationToken?: string,
+): Promise<INodeListSearchResult> {
+	const qs = {
+		pageToken: (paginationToken as string) || undefined,
+	};
+
+	const response = await googleApiRequest.call(this, 'GET', '/v2/projects', undefined, qs);
+
+	let { projects } = response;
+
+	if (filter) {
+		projects = projects.filter(
+			(project: IDataObject) =>
+				(project.friendlyName as string).includes(filter) ||
+				(project.id as string).includes(filter),
+		);
+	}
+
+	return {
+		results: projects.map((project: IDataObject) => ({
+			name: project.friendlyName as string,
+			value: project.id,
+			url: `https://console.cloud.google.com/bigquery?project=${project.id as string}`,
+		})),
+		paginationToken: response.nextPageToken,
+	};
+}
+
+export async function searchDatasets(
+	this: ILoadOptionsFunctions,
+	filter?: string,
+	paginationToken?: string,
+): Promise<INodeListSearchResult> {
+	const projectId = this.getNodeParameter('projectId', undefined, {
+		extractValue: true,
+	});
+
+	const qs = {
+		pageToken: (paginationToken as string) || undefined,
+	};
+
+	const response = await googleApiRequest.call(
+		this,
+		'GET',
+		`/v2/projects/${projectId}/datasets`,
+		undefined,
+		qs,
+	);
+
+	let { datasets } = response;
+
+	if (filter) {
+		datasets = datasets.filter((dataset: { datasetReference: IDataObject }) =>
+			(dataset.datasetReference.datasetId as string).includes(filter),
+		);
+	}
+
+	return {
+		results: datasets.map((dataset: { datasetReference: IDataObject }) => ({
+			name: dataset.datasetReference.datasetId as string,
+			value: dataset.datasetReference.datasetId,
+		})),
+		paginationToken: response.nextPageToken,
+	};
+}
+
+export async function searchTables(
+	this: ILoadOptionsFunctions,
+	filter?: string,
+	paginationToken?: string,
+): Promise<INodeListSearchResult> {
+	const projectId = this.getNodeParameter('projectId', undefined, {
+		extractValue: true,
+	});
+
+	const datasetId = this.getNodeParameter('datasetId', undefined, {
+		extractValue: true,
+	});
+
+	const qs = {
+		pageToken: (paginationToken as string) || undefined,
+	};
+
+	const response = await googleApiRequest.call(
+		this,
+		'GET',
+		`/v2/projects/${projectId}/datasets/${datasetId}/tables`,
+		undefined,
+		qs,
+	);
+
+	let { tables } = response;
+
+	if (filter) {
+		tables = tables.filter((table: { tableReference: IDataObject }) =>
+			(table.tableReference.tableId as string).includes(filter),
+		);
+	}
+
+	const returnData = {
+		results: tables.map((table: { tableReference: IDataObject }) => ({
+			name: table.tableReference.tableId as string,
+			value: table.tableReference.tableId,
+		})),
+		paginationToken: response.nextPageToken,
+	};
+
+	return returnData;
+}

packages/nodes-base/nodes/Google/BigQuery/v2/methods/loadOptions.ts

@@ -0,0 +1,53 @@
+import type { IDataObject, ILoadOptionsFunctions, INodePropertyOptions } from 'n8n-workflow';
+import { googleApiRequest } from '../transport';
+
+export async function getDatasets(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
+	const projectId = this.getNodeParameter('projectId', undefined, {
+		extractValue: true,
+	});
+	const returnData: INodePropertyOptions[] = [];
+	const { datasets } = await googleApiRequest.call(
+		this,
+		'GET',
+		`/v2/projects/${projectId}/datasets`,
+	);
+	for (const dataset of datasets) {
+		returnData.push({
+			name: dataset.datasetReference.datasetId as string,
+			value: dataset.datasetReference.datasetId,
+		});
+	}
+	return returnData;
+}
+
+export async function getSchema(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
+	const projectId = this.getNodeParameter('projectId', undefined, {
+		extractValue: true,
+	});
+	const datasetId = this.getNodeParameter('datasetId', undefined, {
+		extractValue: true,
+	});
+	const tableId = this.getNodeParameter('tableId', undefined, {
+		extractValue: true,
+	});
+
+	const returnData: INodePropertyOptions[] = [];
+
+	const { schema } = await googleApiRequest.call(
+		this,
+		'GET',
+		`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
+		{},
+	);
+
+	for (const field of schema.fields as IDataObject[]) {
+		returnData.push({
+			name: field.name as string,
+			value: field.name as string,
+			// eslint-disable-next-line n8n-nodes-base/node-param-description-lowercase-first-char
+			description:
+				`type: ${field.type as string}` + (field.mode ? ` mode: ${field.mode as string}` : ''),
+		});
+	}
+	return returnData;
+}