feat: Migrate integer primary keys to nanoids (#6345)

* first commit for postgres migration

* (not working)

* sqlite migration

* quicksave

* fix tests

* fix pg test

* fix postgres

* fix variables import

* fix execution saving

* add user settings fix

* change migration to single lines

* patch preferences endpoint

* cleanup

* improve variable import

* cleanup unusued code

* Update packages/cli/src/PublicApi/v1/handlers/workflows/workflows.handler.ts

Co-authored-by: Omar Ajoue <krynble@gmail.com>

* address review notes

* fix var update/import

* refactor: Separate execution data to its own table (#6323)

* wip: Temporary migration process

* refactor: Create boilerplate repository methods for executions

* fix: Lint issues

* refactor: Added search endpoint to repository

* refactor: Make the execution list work again

* wip: Updating how we create and update executions everywhere

* fix: Lint issues and remove most of the direct access to execution model

* refactor: Remove includeWorkflowData flag and fix more tests

* fix: Lint issues

* fix: Fixed ordering of executions for FE, removed transaction when saving execution and removed unnecessary update

* refactor: Add comment about missing feature

* refactor: Refactor counting executions

* refactor: Add migration for other dbms and fix issues found

* refactor: Fix lint issues

* refactor: Remove unnecessary comment and auto inject repo to internal hooks

* refactor: remove type assertion

* fix: Fix broken tests

* fix: Remove unnecessary import

* Remove unnecessary toString() call

Co-authored-by: Iván Ovejero <ivov.src@gmail.com>

* fix: Address comments after review

* refactor: Remove unused import

* fix: Lint issues

* fix: Add correct migration files

---------

Co-authored-by: Iván Ovejero <ivov.src@gmail.com>

* remove null values from credential export

* fix: Fix an issue with queue mode where all running execution would be returned

* fix: Update n8n node to allow for workflow ids with letters

* set upstream on set branch

* remove typo

* add nodeAccess to credentials

* fix unsaved run check for undefined id

* fix(core): Rename version control feature to source control (#6480)

* rename versionControl to sourceControl

* fix source control tooltip wording

---------

Co-authored-by: Romain Minaud <romain.minaud@gmail.com>

* fix(editor): Pay 548 hide the set up version control button (#6485)

* feat(DebugHelper Node): Fix and include in main app (#6406)

* improve node a bit

* fixing continueOnFail() ton contain error in json

* improve pairedItem

* fix random data returning object results

* fix nanoId length typo

* update pnpm-lock file

---------

Co-authored-by: Marcus <marcus@n8n.io>

* fix(editor): Remove setup source control CTA button

* fix(editor): Remove setup source control CTA button

---------

Co-authored-by: Michael Auerswald <michael.auerswald@gmail.com>
Co-authored-by: Marcus <marcus@n8n.io>

* fix(editor): Update source control docs links (#6488)

* feat(DebugHelper Node): Fix and include in main app (#6406)

* improve node a bit

* fixing continueOnFail() ton contain error in json

* improve pairedItem

* fix random data returning object results

* fix nanoId length typo

* update pnpm-lock file

---------

Co-authored-by: Marcus <marcus@n8n.io>

* feat(editor): Replace root events with event bus events (no-changelog) (#6454)

* feat: replace root events with event bus events

* fix: prevent cypress from replacing global with globalThis in import path

* feat: remove emitter mixin

* fix: replace component events with event bus

* fix: fix linting issue

* fix: fix breaking expression switch

* chore: prettify ndv e2e suite code

* fix(editor): Update source control docs links

---------

Co-authored-by: Michael Auerswald <michael.auerswald@gmail.com>
Co-authored-by: Marcus <marcus@n8n.io>
Co-authored-by: Alex Grozav <alex@grozav.com>

* fix tag endpoint regex

---------

Co-authored-by: Omar Ajoue <krynble@gmail.com>
Co-authored-by: Iván Ovejero <ivov.src@gmail.com>
Co-authored-by: Romain Minaud <romain.minaud@gmail.com>
Co-authored-by: Csaba Tuncsik <csaba@n8n.io>
Co-authored-by: Marcus <marcus@n8n.io>
Co-authored-by: Alex Grozav <alex@grozav.com>
This commit is contained in:
Michael Auerswald
2023-06-20 19:13:18 +02:00
committed by GitHub
parent da330f0648
commit c3ba0123ad
156 changed files with 3499 additions and 2594 deletions

View File

@@ -0,0 +1,262 @@
/* eslint-disable n8n-local-rules/no-unneeded-backticks */
import type { MigrationContext, ReversibleMigration } from '@db/types';
export class MigrateIntegerKeysToString1690000000000 implements ReversibleMigration {
async up({ queryRunner, tablePrefix }: MigrationContext) {
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflow_entity RENAME COLUMN id to tmp_id;`,
);
await queryRunner.query(`ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN id varchar(36);`);
await queryRunner.query(`UPDATE ${tablePrefix}workflow_entity SET id = tmp_id::text;`);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflow_entity ALTER COLUMN id SET NOT NULL;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflow_entity ALTER COLUMN tmp_id DROP DEFAULT;`,
);
await queryRunner.query(`DROP SEQUENCE IF EXISTS ${tablePrefix}workflow_entity_id_seq;`);
await queryRunner.query(
`CREATE UNIQUE INDEX "pk_workflow_entity_id" ON ${tablePrefix}workflow_entity ("id");`,
);
await queryRunner.query(`ALTER TABLE ${tablePrefix}tag_entity RENAME COLUMN id to tmp_id;`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}tag_entity ADD COLUMN id varchar(36);`);
await queryRunner.query(`UPDATE ${tablePrefix}tag_entity SET id = tmp_id::text;`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}tag_entity ALTER COLUMN id SET NOT NULL;`);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}tag_entity ALTER COLUMN tmp_id DROP DEFAULT;`,
);
await queryRunner.query(`DROP SEQUENCE IF EXISTS tag_entity_id_seq;`);
await queryRunner.query(
`CREATE UNIQUE INDEX "pk_tag_entity_id" ON ${tablePrefix}tag_entity ("id");`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags RENAME COLUMN "workflowId" to "tmp_workflowId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags ADD COLUMN "workflowId" varchar(36);`,
);
await queryRunner.query(
`UPDATE ${tablePrefix}workflows_tags SET "workflowId" = "tmp_workflowId"::text;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags ALTER COLUMN "workflowId" SET NOT NULL;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags RENAME COLUMN "tagId" to "tmp_tagId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags ADD COLUMN "tagId" varchar(36);`,
);
await queryRunner.query(`UPDATE ${tablePrefix}workflows_tags SET "tagId" = "tmp_tagId"::text;`);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags ALTER COLUMN "tagId" SET NOT NULL;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags DROP CONSTRAINT IF EXISTS "FK_31140eb41f019805b40d0087449";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags DROP CONSTRAINT IF EXISTS "FK_5e29bfe9e22c5d6567f509d4a46";`,
);
await queryRunner.query(
`CREATE UNIQUE INDEX "pk_workflows_tags" ON ${tablePrefix}workflows_tags ("workflowId","tagId");`,
);
await queryRunner.query(`DROP INDEX IF EXISTS "idx_31140eb41f019805b40d008744";`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}workflows_tags DROP CONSTRAINT "PK_a60448a90e51a114e95e2a125b3",
ADD CONSTRAINT "pk_workflows_tags" PRIMARY KEY USING INDEX "pk_workflows_tags";`);
await queryRunner.query(
`CREATE INDEX "idx_workflows_tags_workflow_id" ON ${tablePrefix}workflows_tags ("workflowId");`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags ADD CONSTRAINT "fk_workflows_tags_workflow_id" FOREIGN KEY ("workflowId") REFERENCES workflow_entity(id) ON DELETE CASCADE ON UPDATE NO ACTION;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags ADD CONSTRAINT "fk_workflows_tags_tag_id" FOREIGN KEY ("tagId") REFERENCES tag_entity(id) ON DELETE CASCADE ON UPDATE NO ACTION;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflows_tags DROP COLUMN "tmp_workflowId";`,
);
await queryRunner.query(`ALTER TABLE ${tablePrefix}workflows_tags DROP COLUMN "tmp_tagId";`);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_workflow RENAME COLUMN "workflowId" to "tmp_workflowId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_workflow ADD COLUMN "workflowId" varchar(36);`,
);
await queryRunner.query(
`UPDATE ${tablePrefix}shared_workflow SET "workflowId" = "tmp_workflowId"::text;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_workflow ALTER COLUMN "workflowId" SET NOT NULL;`,
);
await queryRunner.query(
`CREATE UNIQUE INDEX "pk_shared_workflow_id" ON ${tablePrefix}shared_workflow ("userId","workflowId");`,
);
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_65a0933c0f19d278881653bf81d35064";`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}shared_workflow DROP CONSTRAINT "PK_cc5d5a71c7b2591f5154ffb0c785e85e",
ADD CONSTRAINT "pk_shared_workflow_id" PRIMARY KEY USING INDEX "pk_shared_workflow_id";`);
await queryRunner.query(
`CREATE INDEX "idx_shared_workflow_workflow_id" ON ${tablePrefix}shared_workflow ("workflowId");`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_workflow ADD CONSTRAINT "fk_shared_workflow_workflow_id" FOREIGN KEY ("workflowId") REFERENCES workflow_entity(id) ON DELETE CASCADE ON UPDATE NO ACTION;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_workflow DROP COLUMN "tmp_workflowId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflow_statistics RENAME COLUMN "workflowId" to "tmp_workflowId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflow_statistics ADD COLUMN "workflowId" varchar(36);`,
);
await queryRunner.query(
`UPDATE ${tablePrefix}workflow_statistics SET "workflowId" = "tmp_workflowId"::text;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflow_statistics ALTER COLUMN "workflowId" SET NOT NULL;`,
);
await queryRunner.query(
`CREATE UNIQUE INDEX "pk_workflow_statistics" ON ${tablePrefix}workflow_statistics ("workflowId","name");`,
);
await queryRunner.query(`ALTER TABLE ${tablePrefix}workflow_statistics DROP CONSTRAINT IF EXISTS "workflow_statistics_pkey",
ADD CONSTRAINT "pk_workflow_statistics" PRIMARY KEY USING INDEX "pk_workflow_statistics";`);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflow_statistics DROP COLUMN "tmp_workflowId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflow_statistics ADD CONSTRAINT "fk_workflow_statistics_workflow_id" FOREIGN KEY ("workflowId") REFERENCES workflow_entity(id) ON DELETE CASCADE ON UPDATE NO ACTION;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}webhook_entity RENAME COLUMN "workflowId" to "tmp_workflowId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}webhook_entity ADD COLUMN "workflowId" varchar(36);`,
);
await queryRunner.query(
`UPDATE ${tablePrefix}webhook_entity SET "workflowId" = "tmp_workflowId"::text;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}webhook_entity ALTER COLUMN "workflowId" SET NOT NULL;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}webhook_entity DROP COLUMN "tmp_workflowId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}webhook_entity ADD CONSTRAINT "fk_webhook_entity_workflow_id" FOREIGN KEY ("workflowId") REFERENCES workflow_entity(id) ON DELETE CASCADE ON UPDATE NO ACTION;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}execution_entity RENAME COLUMN "workflowId" to "tmp_workflowId";`,
);
// -- Intentionally NOT setting colum to NOT NULL
await queryRunner.query(
`ALTER TABLE ${tablePrefix}execution_entity ADD COLUMN "workflowId" varchar(36);`,
);
await queryRunner.query(
`UPDATE ${tablePrefix}execution_entity SET "workflowId" = "tmp_workflowId"::text;`,
);
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_d160d4771aba5a0d78943edbe3";`);
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_4f474ac92be81610439aaad61e";`);
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_58154df94c686818c99fb754ce";`);
// -- index idx_33228da131bb1112247cf52a42 is a duplicate of IDX_33228da131bb1112247cf52a42
await queryRunner.query(`DROP INDEX IF EXISTS "idx_33228da131bb1112247cf52a42";`);
await queryRunner.query(
`CREATE INDEX "idx_execution_entity_workflow_id_id" ON ${tablePrefix}execution_entity ("workflowId","id");`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}execution_entity DROP COLUMN "tmp_workflowId";`,
);
// -- FK was missing in prev schema - should it be added?
await queryRunner.query(
`ALTER TABLE ${tablePrefix}execution_entity ADD CONSTRAINT "fk_execution_entity_workflow_id" FOREIGN KEY ("workflowId") REFERENCES workflow_entity(id) ON DELETE CASCADE ON UPDATE NO ACTION;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}workflow_entity DROP CONSTRAINT IF EXISTS "pk_eded7d72664448da7745d551207";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}tag_entity DROP CONSTRAINT IF EXISTS "PK_7a50a9b74ae6855c0dcaee25052";`,
);
await queryRunner.query(`ALTER TABLE ${tablePrefix}workflow_entity DROP COLUMN tmp_id;`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}tag_entity DROP COLUMN tmp_id;`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}workflow_entity ADD PRIMARY KEY (id);`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}tag_entity ADD PRIMARY KEY (id);`);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}credentials_entity RENAME COLUMN id to tmp_id;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}credentials_entity ADD COLUMN id varchar(36);`,
);
await queryRunner.query(`UPDATE ${tablePrefix}credentials_entity SET id = tmp_id::text;`);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}credentials_entity ALTER COLUMN id SET NOT NULL;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}credentials_entity ALTER COLUMN tmp_id DROP DEFAULT;`,
);
await queryRunner.query(`DROP SEQUENCE IF EXISTS credentials_entity_id_seq;`);
await queryRunner.query(
`CREATE UNIQUE INDEX "pk_credentials_entity_id" ON ${tablePrefix}credentials_entity ("id");`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_credentials RENAME COLUMN "credentialsId" to "tmp_credentialsId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_credentials ADD COLUMN "credentialsId" varchar(36);`,
);
await queryRunner.query(
`UPDATE ${tablePrefix}shared_credentials SET "credentialsId" = "tmp_credentialsId"::text;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_credentials ALTER COLUMN "credentialsId" SET NOT NULL;`,
);
await queryRunner.query(
`CREATE UNIQUE INDEX "pk_shared_credentials_id" ON ${tablePrefix}shared_credentials ("userId","credentialsId");`,
);
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_829d16efa0e265cb076d50eca8d21733";`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}shared_credentials DROP CONSTRAINT "PK_10dd1527ffb639609be7aadd98f628c6",
ADD CONSTRAINT "pk_shared_credentials_id" PRIMARY KEY USING INDEX "pk_shared_credentials_id";`);
await queryRunner.query(
`CREATE INDEX "idx_shared_credentials_credentials_id" ON ${tablePrefix}shared_credentials ("credentialsId");`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_credentials ADD CONSTRAINT "fk_shared_credentials_credentials_id" FOREIGN KEY ("credentialsId") REFERENCES credentials_entity(id) ON DELETE CASCADE ON UPDATE NO ACTION;`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}shared_credentials DROP COLUMN "tmp_credentialsId";`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}credentials_entity DROP CONSTRAINT IF EXISTS "pk_814c3d3c36e8a27fa8edb761b0e";`,
);
await queryRunner.query(`ALTER TABLE ${tablePrefix}credentials_entity DROP COLUMN tmp_id;`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}credentials_entity ADD PRIMARY KEY (id);`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}variables RENAME COLUMN id to tmp_id;`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}variables ADD COLUMN id varchar(36);`);
await queryRunner.query(`UPDATE ${tablePrefix}variables SET id = tmp_id::text;`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}variables ALTER COLUMN id SET NOT NULL;`);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}variables ALTER COLUMN tmp_id DROP DEFAULT;`,
);
await queryRunner.query(`DROP SEQUENCE IF EXISTS variables_id_seq;`);
await queryRunner.query(
`CREATE UNIQUE INDEX "pk_variables_id" ON ${tablePrefix}variables ("id");`,
);
await queryRunner.query(
`ALTER TABLE ${tablePrefix}variables DROP CONSTRAINT IF EXISTS "variables_pkey";`,
);
await queryRunner.query(`ALTER TABLE ${tablePrefix}variables DROP COLUMN tmp_id;`);
await queryRunner.query(`ALTER TABLE ${tablePrefix}variables ADD PRIMARY KEY (id);`);
}
// eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars
async down({ queryRunner, tablePrefix }: MigrationContext) {}
}

View File

@@ -0,0 +1,42 @@
import type { MigrationContext, ReversibleMigration } from '@db/types';
export class SeparateExecutionData1690000000020 implements ReversibleMigration {
async up({ queryRunner, tablePrefix }: MigrationContext) {
await queryRunner.query(
`CREATE TABLE "${tablePrefix}execution_data" (
"executionId" integer NOT NULL,
"workflowData" json NOT NULL,
"data" text NOT NULL,
CONSTRAINT "${tablePrefix}execution_data_fk" FOREIGN KEY ("executionId") REFERENCES ${tablePrefix}execution_entity(id) ON DELETE CASCADE
)`,
);
await queryRunner.query(
`INSERT INTO "${tablePrefix}execution_data" (
"executionId",
"workflowData",
"data")
SELECT "id", "workflowData", "data" FROM "${tablePrefix}execution_entity"
`,
);
await queryRunner.query(
`ALTER TABLE "${tablePrefix}execution_entity" DROP COLUMN "workflowData", DROP COLUMN "data"`,
);
}
async down({ queryRunner, tablePrefix }: MigrationContext) {
await queryRunner.query(
`ALTER TABLE "${tablePrefix}execution_entity"
ADD "workflowData" json NULL,
ADD "data" text NULL`,
);
await queryRunner.query(
`UPDATE "${tablePrefix}execution_entity" SET "workflowData" = "execution_data"."workflowData", "data" = "execution_data"."data"
FROM "${tablePrefix}execution_data" WHERE "${tablePrefix}execution_data"."executionId" = "${tablePrefix}execution_entity"."id"`,
);
await queryRunner.query(`DROP TABLE "${tablePrefix}execution_data"`);
}
}

View File

@@ -36,6 +36,8 @@ import { UpdateRunningExecutionStatus1677236854063 } from './1677236854063-Updat
import { CreateExecutionMetadataTable1679416281778 } from './1679416281778-CreateExecutionMetadataTable';
import { CreateVariables1677501636754 } from './1677501636754-CreateVariables';
import { AddUserActivatedProperty1681134145996 } from './1681134145996-AddUserActivatedProperty';
import { MigrateIntegerKeysToString1690000000000 } from './1690000000000-MigrateIntegerKeysToString';
import { SeparateExecutionData1690000000020 } from './1690000000020-SeparateExecutionData';
export const postgresMigrations: Migration[] = [
InitialMigration1587669153312,
@@ -75,4 +77,6 @@ export const postgresMigrations: Migration[] = [
CreateExecutionMetadataTable1679416281778,
CreateVariables1677501636754,
AddUserActivatedProperty1681134145996,
MigrateIntegerKeysToString1690000000000,
SeparateExecutionData1690000000020,
];