Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Biigint ftw wave 2 #9170

Merged
merged 6 commits into from
Dec 5, 2024
Merged

Biigint ftw wave 2 #9170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
85d541c
BigInt for connectors DB
flvndvd Dec 5, 2024
59cf356
✨
flvndvd Dec 5, 2024
c249e48
tmp
flvndvd Dec 5, 2024
b4ab630
Fix connectors foreign key constraint
flvndvd Dec 5, 2024
36f3517
Fix safety flag on migration
flvndvd Dec 5, 2024
81f538e
👕
flvndvd Dec 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions connectors/migrations/db/migration_40.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
\set ON_ERROR_STOP on

-- Check if the safety flag variable exists
\if :i_know_what_i_do
-- your migration code here
\else
\echo 'Safety flag not set. Run with: psql -v i_know_what_i_do=true -f migration_40.sql'
\q
\endif

DO $$
DECLARE
r RECORD;
BEGIN
-- First convert all PKs
FOR r IN (
SELECT
t.table_name,
c.column_name
FROM information_schema.tables t
JOIN information_schema.columns c
ON t.table_name = c.table_name
AND t.table_schema = c.table_schema
JOIN information_schema.key_column_usage k
ON c.column_name = k.column_name
AND c.table_name = k.table_name
WHERE c.data_type = 'integer'
AND t.table_schema = 'public'
AND EXISTS (
SELECT 1
FROM information_schema.table_constraints tc
WHERE tc.constraint_type = 'PRIMARY KEY'
AND tc.table_name = t.table_name
AND tc.constraint_name = k.constraint_name
)
) LOOP
RAISE NOTICE 'Converting PK column % in table % to bigint', r.column_name, r.table_name;
EXECUTE format('ALTER TABLE %I ALTER COLUMN %I TYPE bigint', r.table_name, r.column_name);
END LOOP;

-- Then convert all FKs
FOR r IN (
SELECT DISTINCT
t.table_name,
c.column_name
FROM information_schema.tables t
JOIN information_schema.columns c
ON t.table_name = c.table_name
AND t.table_schema = c.table_schema
JOIN information_schema.key_column_usage k
ON c.column_name = k.column_name
AND c.table_name = k.table_name
JOIN information_schema.table_constraints tc
ON tc.constraint_name = k.constraint_name
WHERE c.data_type = 'integer'
AND t.table_schema = 'public'
AND tc.constraint_type = 'FOREIGN KEY'
) LOOP
RAISE NOTICE 'Converting FK column % in table % to bigint', r.column_name, r.table_name;
EXECUTE format('ALTER TABLE %I ALTER COLUMN %I TYPE bigint', r.table_name, r.column_name);
END LOOP;
END $$;
52 changes: 10 additions & 42 deletions connectors/src/lib/models/confluence.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,11 @@
import type {
CreationOptional,
ForeignKey,
InferAttributes,
InferCreationAttributes,
} from "sequelize";
import { DataTypes, Model } from "sequelize";
import type { CreationOptional, ForeignKey } from "sequelize";
import { DataTypes } from "sequelize";

import { sequelizeConnection } from "@connectors/resources/storage";
import { ConnectorModel } from "@connectors/resources/storage/models/connector_model";
import { BaseModel } from "@connectors/resources/storage/wrappers";

export class ConfluenceConfiguration extends Model<
InferAttributes<ConfluenceConfiguration>,
InferCreationAttributes<ConfluenceConfiguration>
> {
declare id: CreationOptional<number>;
export class ConfluenceConfiguration extends BaseModel<ConfluenceConfiguration> {
declare createdAt: CreationOptional<Date>;
declare updatedAt: CreationOptional<Date>;

Expand All @@ -25,11 +17,6 @@ export class ConfluenceConfiguration extends Model<
}
ConfluenceConfiguration.init(
{
id: {
type: DataTypes.INTEGER,
autoIncrement: true,
primaryKey: true,
},
cloudId: {
type: DataTypes.STRING,
allowNull: false,
Expand Down Expand Up @@ -65,11 +52,7 @@ ConfluenceConfiguration.init(
ConnectorModel.hasOne(ConfluenceConfiguration);

// ConfluenceSpace stores the global spaces selected by the user to sync.
export class ConfluenceSpace extends Model<
InferAttributes<ConfluenceSpace>,
InferCreationAttributes<ConfluenceSpace>
> {
declare id: CreationOptional<number>;
export class ConfluenceSpace extends BaseModel<ConfluenceSpace> {
declare createdAt: CreationOptional<Date>;
declare updatedAt: CreationOptional<Date>;
declare connectorId: ForeignKey<ConnectorModel["id"]>;
Expand All @@ -79,15 +62,6 @@ export class ConfluenceSpace extends Model<
}
ConfluenceSpace.init(
{
id: {
type: DataTypes.INTEGER,
autoIncrement: true,
primaryKey: true,
},
connectorId: {
type: DataTypes.INTEGER,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not needed since we have a relationship defined below.

allowNull: false,
},
createdAt: {
type: DataTypes.DATE,
allowNull: false,
Expand Down Expand Up @@ -117,14 +91,13 @@ ConfluenceSpace.init(
indexes: [{ fields: ["connectorId", "spaceId"], unique: true }],
}
);
ConnectorModel.hasOne(ConfluenceSpace);
ConnectorModel.hasOne(ConfluenceSpace, {
foreignKey: "connectorId",
onDelete: "RESTRICT",
});

// ConfluencePages stores the pages.
export class ConfluencePage extends Model<
InferAttributes<ConfluencePage>,
InferCreationAttributes<ConfluencePage>
> {
declare id: CreationOptional<number>;
export class ConfluencePage extends BaseModel<ConfluencePage> {
declare createdAt: CreationOptional<Date>;
declare updatedAt: CreationOptional<Date>;
declare lastVisitedAt: CreationOptional<Date>;
Expand All @@ -141,11 +114,6 @@ export class ConfluencePage extends Model<
}
ConfluencePage.init(
{
id: {
type: DataTypes.INTEGER,
autoIncrement: true,
primaryKey: true,
},
createdAt: {
type: DataTypes.DATE,
allowNull: false,
Expand Down
76 changes: 9 additions & 67 deletions connectors/src/lib/models/github.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,11 @@
import type {
CreationOptional,
ForeignKey,
InferAttributes,
InferCreationAttributes,
} from "sequelize";
import { DataTypes, Model } from "sequelize";
import type { CreationOptional, ForeignKey } from "sequelize";
import { DataTypes } from "sequelize";

import { sequelizeConnection } from "@connectors/resources/storage";
import { ConnectorModel } from "@connectors/resources/storage/models/connector_model";
import { BaseModel } from "@connectors/resources/storage/wrappers";

export class GithubConnectorState extends Model<
InferAttributes<GithubConnectorState>,
InferCreationAttributes<GithubConnectorState>
> {
declare id: CreationOptional<number>;
export class GithubConnectorState extends BaseModel<GithubConnectorState> {
declare createdAt: CreationOptional<Date>;
declare updatedAt: CreationOptional<Date>;

Expand All @@ -25,11 +17,6 @@ export class GithubConnectorState extends Model<
}
GithubConnectorState.init(
{
id: {
type: DataTypes.INTEGER,
autoIncrement: true,
primaryKey: true,
},
createdAt: {
type: DataTypes.DATE,
allowNull: false,
Expand Down Expand Up @@ -65,11 +52,7 @@ GithubConnectorState.init(
);
ConnectorModel.hasOne(GithubConnectorState);

export class GithubIssue extends Model<
InferAttributes<GithubIssue>,
InferCreationAttributes<GithubIssue>
> {
declare id: CreationOptional<number>;
export class GithubIssue extends BaseModel<GithubIssue> {
declare createdAt: CreationOptional<Date>;
declare updatedAt: CreationOptional<Date>;

Expand All @@ -82,11 +65,6 @@ export class GithubIssue extends Model<
}
GithubIssue.init(
{
id: {
type: DataTypes.INTEGER,
autoIncrement: true,
primaryKey: true,
},
createdAt: {
type: DataTypes.DATE,
allowNull: false,
Expand Down Expand Up @@ -122,11 +100,7 @@ GithubIssue.init(
);
ConnectorModel.hasMany(GithubIssue);

export class GithubDiscussion extends Model<
InferAttributes<GithubDiscussion>,
InferCreationAttributes<GithubDiscussion>
> {
declare id: CreationOptional<number>;
export class GithubDiscussion extends BaseModel<GithubDiscussion> {
declare createdAt: CreationOptional<Date>;
declare updatedAt: CreationOptional<Date>;

Expand All @@ -137,11 +111,6 @@ export class GithubDiscussion extends Model<
}
GithubDiscussion.init(
{
id: {
type: DataTypes.INTEGER,
autoIncrement: true,
primaryKey: true,
},
createdAt: {
type: DataTypes.DATE,
allowNull: false,
Expand Down Expand Up @@ -173,11 +142,7 @@ GithubDiscussion.init(
);
ConnectorModel.hasMany(GithubDiscussion);

export class GithubCodeRepository extends Model<
InferAttributes<GithubCodeRepository>,
InferCreationAttributes<GithubCodeRepository>
> {
declare id: CreationOptional<number>;
export class GithubCodeRepository extends BaseModel<GithubCodeRepository> {
declare createdAt: CreationOptional<Date>;
declare updatedAt: CreationOptional<Date>;
declare lastSeenAt: CreationOptional<Date>;
Expand All @@ -194,11 +159,6 @@ export class GithubCodeRepository extends Model<
}
GithubCodeRepository.init(
{
id: {
type: DataTypes.INTEGER,
autoIncrement: true,
primaryKey: true,
},
createdAt: {
type: DataTypes.DATE,
allowNull: false,
Expand Down Expand Up @@ -249,11 +209,7 @@ GithubCodeRepository.init(
);
ConnectorModel.hasMany(GithubCodeRepository);

export class GithubCodeFile extends Model<
InferAttributes<GithubCodeFile>,
InferCreationAttributes<GithubCodeFile>
> {
declare id: CreationOptional<number>;
export class GithubCodeFile extends BaseModel<GithubCodeFile> {
declare createdAt: CreationOptional<Date>;
declare updatedAt: CreationOptional<Date>;
declare lastSeenAt: CreationOptional<Date>;
Expand All @@ -271,11 +227,6 @@ export class GithubCodeFile extends Model<
}
GithubCodeFile.init(
{
id: {
type: DataTypes.INTEGER,
autoIncrement: true,
primaryKey: true,
},
createdAt: {
type: DataTypes.DATE,
allowNull: false,
Expand Down Expand Up @@ -332,11 +283,7 @@ GithubCodeFile.init(
);
ConnectorModel.hasMany(GithubCodeFile);

export class GithubCodeDirectory extends Model<
InferAttributes<GithubCodeDirectory>,
InferCreationAttributes<GithubCodeDirectory>
> {
declare id: CreationOptional<number>;
export class GithubCodeDirectory extends BaseModel<GithubCodeDirectory> {
declare createdAt: CreationOptional<Date>;
declare updatedAt: CreationOptional<Date>;
declare lastSeenAt: CreationOptional<Date>;
Expand All @@ -353,11 +300,6 @@ export class GithubCodeDirectory extends Model<
}
GithubCodeDirectory.init(
{
id: {
type: DataTypes.INTEGER,
autoIncrement: true,
primaryKey: true,
},
createdAt: {
type: DataTypes.DATE,
allowNull: false,
Expand Down
Loading
Loading