From b5e258f80dda9984b7b3077bbd7da6f1ea21116d Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Mon, 21 Jul 2025 13:04:03 -0400 Subject: [PATCH 001/164] Empty From 1434d05ebdbb55dd3fd5d04834a3e1fc48b8b1bf Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Mon, 21 Jul 2025 13:04:28 -0400 Subject: [PATCH 002/164] Empty From 2f13cb033de32064cd6cfa8345f812f145629c1b Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Mon, 21 Jul 2025 14:34:31 -0400 Subject: [PATCH 003/164] Basic UI --- frontend/src/metabase-types/api/index.ts | 1 + frontend/src/metabase-types/api/transform.ts | 29 ++++++++++ frontend/src/metabase/api/index.ts | 1 + frontend/src/metabase/api/tags/constants.ts | 1 + frontend/src/metabase/api/tags/utils.ts | 13 +++++ frontend/src/metabase/api/transform.ts | 60 ++++++++++++++++++++ 6 files changed, 105 insertions(+) create mode 100644 frontend/src/metabase-types/api/transform.ts create mode 100644 frontend/src/metabase/api/transform.ts diff --git a/frontend/src/metabase-types/api/index.ts b/frontend/src/metabase-types/api/index.ts index 2290768bd3d4..da645b5400be 100644 --- a/frontend/src/metabase-types/api/index.ts +++ b/frontend/src/metabase-types/api/index.ts @@ -49,6 +49,7 @@ export * from "./subscription"; export * from "./table"; export * from "./task"; export * from "./timeline"; +export * from "./transform"; export * from "./user"; export * from "./util"; export * from "./visualization"; diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts new file mode 100644 index 000000000000..f1d941bff2cb --- /dev/null +++ b/frontend/src/metabase-types/api/transform.ts @@ -0,0 +1,29 @@ +import type { DatabaseId } from "./database"; +import type { DatasetQuery } from "./query"; + +export type TransformId = number; + +export type Transform = { + id: TransformId; + name: string; + source: TransformSource; + target: TransformTarget; +}; + +export type TransformSource = { + type: "query"; + query: DatasetQuery; +}; + +export type TransformTarget = { + type: "table"; + database: DatabaseId; + schema: string; + table: string; +}; + +export type CreateTransformRequest = { + name: string; + source: TransformSource; + target: TransformTarget; +}; diff --git a/frontend/src/metabase/api/index.ts b/frontend/src/metabase/api/index.ts index 0b6a9fc0c055..217457a97009 100644 --- a/frontend/src/metabase/api/index.ts +++ b/frontend/src/metabase/api/index.ts @@ -35,5 +35,6 @@ export * from "./table"; export * from "./task"; export * from "./timeline"; export * from "./timeline-event"; +export * from "./transform"; export * from "./user-key-value"; export * from "./user"; diff --git a/frontend/src/metabase/api/tags/constants.ts b/frontend/src/metabase/api/tags/constants.ts index 87a6ee00dad7..135f6e18cdaa 100644 --- a/frontend/src/metabase/api/tags/constants.ts +++ b/frontend/src/metabase/api/tags/constants.ts @@ -34,6 +34,7 @@ export const TAG_TYPES = [ "task", "timeline", "timeline-event", + "transform", "user", "public-dashboard", "embed-dashboard", diff --git a/frontend/src/metabase/api/tags/utils.ts b/frontend/src/metabase/api/tags/utils.ts index 8612c53920a9..ca052717727f 100644 --- a/frontend/src/metabase/api/tags/utils.ts +++ b/frontend/src/metabase/api/tags/utils.ts @@ -40,6 +40,7 @@ import type { Task, Timeline, TimelineEvent, + Transform, UserInfo, WritebackAction, } from "metabase-types/api"; @@ -605,6 +606,18 @@ export function provideTaskListTags(tasks: Task[]): TagDescription[] { return [listTag("task"), ...tasks.flatMap(provideTaskTags)]; } +export function provideTransformTags( + transform: Transform, +): TagDescription[] { + return [idTag("transform", transform.id)]; +} + +export function provideTransformListTags( + transforms: Transform[], +): TagDescription[] { + return [listTag("transform"), ...transforms.flatMap(provideTransformTags)]; +} + export function provideUniqueTasksListTags(): TagDescription[] { return [listTag("unique-tasks")]; } diff --git a/frontend/src/metabase/api/transform.ts b/frontend/src/metabase/api/transform.ts new file mode 100644 index 000000000000..caed74c33e10 --- /dev/null +++ b/frontend/src/metabase/api/transform.ts @@ -0,0 +1,60 @@ +import type { + CreateTransformRequest, + Transform, + TransformId, +} from "metabase-types/api"; + +import { Api } from "./api"; +import { + idTag, + invalidateTags, + listTag, + provideTransformListTags, + provideTransformTags, + tag, +} from "./tags"; + +export const transformApi = Api.injectEndpoints({ + endpoints: (builder) => ({ + listTransforms: builder.query({ + query: (params) => ({ + method: "GET", + url: "/api/transform", + params, + }), + providesTags: (transforms = []) => provideTransformListTags(transforms), + }), + getTransform: builder.query({ + query: (id) => ({ + method: "GET", + url: `/api/transform/${id}`, + }), + providesTags: (transform) => + transform ? provideTransformTags(transform) : [], + }), + createTransform: builder.mutation({ + query: (body) => ({ + method: "POST", + url: "/api/transform", + body, + }), + invalidatesTags: (_, error) => + invalidateTags(error, [listTag("transform"), tag("transform")]), + }), + deleteTransform: builder.mutation({ + query: (id) => ({ + method: "DELETE", + url: `/api/transform/${id}`, + }), + invalidatesTags: (_, error, id) => + invalidateTags(error, [listTag("transform"), idTag("transform", id)]), + }), + }), +}); + +export const { + useListTransformsQuery, + useGetTransformQuery, + useCreateTransformMutation, + useDeleteTransformMutation, +} = transformApi; From 87ed3b59650b75eab94142a095474624f8992ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Mon, 21 Jul 2025 21:46:40 +0300 Subject: [PATCH 004/164] Add metabase-enterprise.transform module --- .clj-kondo/config/modules/config.edn | 6 ++++++ enterprise/backend/src/metabase_enterprise/core/init.clj | 3 ++- .../backend/src/metabase_enterprise/transforms/api.clj | 1 + .../backend/src/metabase_enterprise/transforms/core.clj | 2 ++ .../backend/src/metabase_enterprise/transforms/init.clj | 3 +++ .../src/metabase_enterprise/transforms/models/transform.clj | 1 + 6 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 enterprise/backend/src/metabase_enterprise/transforms/api.clj create mode 100644 enterprise/backend/src/metabase_enterprise/transforms/core.clj create mode 100644 enterprise/backend/src/metabase_enterprise/transforms/init.clj create mode 100644 enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn index aecf2af31da5..9c650c52668f 100644 --- a/.clj-kondo/config/modules/config.edn +++ b/.clj-kondo/config/modules/config.edn @@ -1830,6 +1830,12 @@ settings util}} + enterprise/transform + {:team "Querying" + :api #{metabase-enterprise.transform.api + metabase-enterprise.transform.init} + :uses #{}} + enterprise/upload-management {:team "Admin Webapp" :api #{metabase-enterprise.upload-management.api} diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj index b0d7ad90ae1e..6b1fbf73cebc 100644 --- a/enterprise/backend/src/metabase_enterprise/core/init.clj +++ b/enterprise/backend/src/metabase_enterprise/core/init.clj @@ -15,4 +15,5 @@ [metabase-enterprise.metabot-v3.init] [metabase-enterprise.scim.init] [metabase-enterprise.sso.init] - [metabase-enterprise.stale.init])) + [metabase-enterprise.stale.init] + [metabase-enterprise.transform.init])) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj new file mode 100644 index 000000000000..a24c839cbfc5 --- /dev/null +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -0,0 +1 @@ +(ns metabase-enterprise.transforms.api) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.clj new file mode 100644 index 000000000000..3d60af71532e --- /dev/null +++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj @@ -0,0 +1,2 @@ +(ns metabase-enterprise.transforms.core + "API namespace for the `metabase-enterprise.transform` module.") diff --git a/enterprise/backend/src/metabase_enterprise/transforms/init.clj b/enterprise/backend/src/metabase_enterprise/transforms/init.clj new file mode 100644 index 000000000000..e1517a954cbd --- /dev/null +++ b/enterprise/backend/src/metabase_enterprise/transforms/init.clj @@ -0,0 +1,3 @@ +(ns metabase-enterprise.transforms.init + (:require + [metabase-enterprise.metabot-v3.settings])) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj new file mode 100644 index 000000000000..44551161e9f4 --- /dev/null +++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj @@ -0,0 +1 @@ +(ns metabase-enterprise.transforms.models.transform) From 117e92366335e642314a14ba14ff3c646007fd49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Mon, 21 Jul 2025 21:52:25 +0300 Subject: [PATCH 005/164] Refer to module with plural --- .clj-kondo/config/modules/config.edn | 6 +++--- enterprise/backend/src/metabase_enterprise/core/init.clj | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn index 9c650c52668f..dc5e00c816af 100644 --- a/.clj-kondo/config/modules/config.edn +++ b/.clj-kondo/config/modules/config.edn @@ -1830,10 +1830,10 @@ settings util}} - enterprise/transform + enterprise/transforms {:team "Querying" - :api #{metabase-enterprise.transform.api - metabase-enterprise.transform.init} + :api #{metabase-enterprise.transforms.api + metabase-enterprise.transforms.init} :uses #{}} enterprise/upload-management diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj index 6b1fbf73cebc..005a18de0fdb 100644 --- a/enterprise/backend/src/metabase_enterprise/core/init.clj +++ b/enterprise/backend/src/metabase_enterprise/core/init.clj @@ -16,4 +16,4 @@ [metabase-enterprise.scim.init] [metabase-enterprise.sso.init] [metabase-enterprise.stale.init] - [metabase-enterprise.transform.init])) + [metabase-enterprise.transforms.init])) From 64f799a04604297ad91814e5847553f5b391ad10 Mon Sep 17 00:00:00 2001 From: William Zimrin Date: Mon, 21 Jul 2025 13:52:42 -0500 Subject: [PATCH 006/164] add dummy execute function --- .../backend/src/metabase_enterprise/transforms/execute.clj | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 enterprise/backend/src/metabase_enterprise/transforms/execute.clj diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj new file mode 100644 index 000000000000..701c480e2ec6 --- /dev/null +++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj @@ -0,0 +1,4 @@ +(ns metabase-enterprise.transforms.execute) + +(defn execute [{:keys [db driver sql output-table overwrite?]}] + (or output-table (str "transform_" (random-uuid)))) From 09ec30117fae88ba62319ec7008f9090e4231c5c Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Mon, 21 Jul 2025 15:00:02 -0400 Subject: [PATCH 007/164] Add a turn into transform modal --- frontend/src/metabase-lib/database.ts | 8 ++ .../components/QueryModals/QueryModals.tsx | 9 ++ .../QuestionMoreActionsMenu.tsx | 11 ++ .../src/metabase/query_builder/constants.ts | 1 + .../NewTransformModal/NewTransformModal.tsx | 109 ++++++++++++++++++ .../components/NewTransformModal/index.ts | 1 + 6 files changed, 139 insertions(+) create mode 100644 frontend/src/metabase/transforms/components/NewTransformModal/NewTransformModal.tsx create mode 100644 frontend/src/metabase/transforms/components/NewTransformModal/index.ts diff --git a/frontend/src/metabase-lib/database.ts b/frontend/src/metabase-lib/database.ts index 7e40adff373a..28372eab9b1a 100644 --- a/frontend/src/metabase-lib/database.ts +++ b/frontend/src/metabase-lib/database.ts @@ -11,3 +11,11 @@ import type { Query } from "./types"; export function databaseID(query: Query): number | null { return ML.database_id(query); } + +export function databaseIdOrThrow(query: Query): number { + const id = databaseID(query); + if (id == null) { + throw new TypeError(`Expected database id, but got ${id}`); + } + return id; +} diff --git a/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx b/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx index c1b4a71b77fc..59268495c70e 100644 --- a/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx +++ b/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx @@ -23,6 +23,7 @@ import ArchiveQuestionModal from "metabase/questions/containers/ArchiveQuestionM import EditEventModal from "metabase/timelines/questions/containers/EditEventModal"; import MoveEventModal from "metabase/timelines/questions/containers/MoveEventModal"; import NewEventModal from "metabase/timelines/questions/containers/NewEventModal"; +import { NewTransformModal } from "metabase/transforms/components/NewTransformModal"; import type Question from "metabase-lib/v1/Question"; import type { Card, DashboardTabId } from "metabase-types/api"; import type { QueryBuilderMode } from "metabase-types/store"; @@ -337,5 +338,13 @@ export function QueryModals({ return ( ); + case MODAL_TYPES.NEW_TRANSFORM: + return ( + + ); } } diff --git a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx index c30b3483b55f..27f41fc61843 100644 --- a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx +++ b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx @@ -34,6 +34,7 @@ const MOVE_TESTID = "move-button"; const TURN_INTO_DATASET_TESTID = "turn-into-dataset"; const CLONE_TESTID = "clone-button"; const ARCHIVE_TESTID = "archive-button"; +const TRANSFORM_TESTID = "transform"; type QuestionMoreActionsMenuProps = { question: Question; @@ -152,6 +153,16 @@ export const QuestionMoreActionsMenu = ({ {t`Turn into a model`} ), + hasDataPermissions && ( + } + data-testid={TRANSFORM_TESTID} + onClick={() => onOpenModal(MODAL_TYPES.NEW_TRANSFORM)} + > + {t`Turn into a transform`} + + ), hasCollectionPermissions && isModel && ( void; +}; + +export function NewTransformModal({ + query, + opened, + onClose, +}: NewTransformModalProps) { + return ( + + + + + {t`New transform`} + + + + + + + + + + ); +} + +type NewTransformFormProps = { + query: Lib.Query; + onClose: () => void; +}; + +type NewTransformSettings = { + name: string; + schema: string; + table: string; +}; + +const NEW_TRANSFORM_SCHEMA = Yup.object().shape({ + name: Yup.string().required(Errors.required).default(""), + schema: Yup.string().required(Errors.required).default(""), + table: Yup.string().required(Errors.required).default(""), +}); + +function NewTransformForm({ query, onClose }: NewTransformFormProps) { + const [createTransform] = useCreateTransformMutation(); + + const handleSubmit = async (settings: NewTransformSettings) => { + await createTransform(getRequest(query, settings)).unwrap(); + onClose(); + }; + + return ( + +
+ + + + + + + + + +
+
+ ); +} + +function getRequest( + query: Lib.Query, + settings: NewTransformSettings, +): CreateTransformRequest { + return { + name: settings.name, + source: { + type: "query", + query: Lib.toLegacyQuery(query), + }, + target: { + type: "table", + database: Lib.databaseIdOrThrow(query), + schema: settings.schema, + table: settings.table, + }, + }; +} diff --git a/frontend/src/metabase/transforms/components/NewTransformModal/index.ts b/frontend/src/metabase/transforms/components/NewTransformModal/index.ts new file mode 100644 index 000000000000..60cee20bd3aa --- /dev/null +++ b/frontend/src/metabase/transforms/components/NewTransformModal/index.ts @@ -0,0 +1 @@ +export * from "./NewTransformModal"; From 5f60c16f307d300d2d8a25baa1a0cf20eb43c643 Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Mon, 21 Jul 2025 15:25:00 -0400 Subject: [PATCH 008/164] Transforms page --- frontend/src/metabase/api/transform.ts | 13 ++ .../browse/transforms/BrowseTransforms.tsx | 59 +++++ .../browse/transforms/TransformsTable.tsx | 213 ++++++++++++++++++ .../src/metabase/browse/transforms/index.ts | 1 + .../src/metabase/browse/transforms/types.ts | 1 + .../MainNavbarContainer/BrowseNavSection.tsx | 13 ++ .../QuestionMoreActionsMenu.tsx | 2 +- frontend/src/metabase/routes.jsx | 2 + 8 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 frontend/src/metabase/browse/transforms/BrowseTransforms.tsx create mode 100644 frontend/src/metabase/browse/transforms/TransformsTable.tsx create mode 100644 frontend/src/metabase/browse/transforms/index.ts create mode 100644 frontend/src/metabase/browse/transforms/types.ts diff --git a/frontend/src/metabase/api/transform.ts b/frontend/src/metabase/api/transform.ts index caed74c33e10..d9e39bc6effe 100644 --- a/frontend/src/metabase/api/transform.ts +++ b/frontend/src/metabase/api/transform.ts @@ -41,6 +41,18 @@ export const transformApi = Api.injectEndpoints({ invalidatesTags: (_, error) => invalidateTags(error, [listTag("transform"), tag("transform")]), }), + executeTransform: builder.mutation({ + query: (id) => ({ + method: "POST", + url: `/api/transform/${id}/execute`, + }), + invalidatesTags: (_, error) => + invalidateTags(error, [ + tag("table"), + tag("field"), + tag("field-values"), + ]), + }), deleteTransform: builder.mutation({ query: (id) => ({ method: "DELETE", @@ -56,5 +68,6 @@ export const { useListTransformsQuery, useGetTransformQuery, useCreateTransformMutation, + useExecuteTransformMutation, useDeleteTransformMutation, } = transformApi; diff --git a/frontend/src/metabase/browse/transforms/BrowseTransforms.tsx b/frontend/src/metabase/browse/transforms/BrowseTransforms.tsx new file mode 100644 index 000000000000..71f0f2618d2a --- /dev/null +++ b/frontend/src/metabase/browse/transforms/BrowseTransforms.tsx @@ -0,0 +1,59 @@ +import { t } from "ttag"; + +import { useListTransformsQuery } from "metabase/api"; +import { DelayedLoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper/DelayedLoadingAndErrorWrapper"; +import { Flex, Group, Icon, Stack, Title } from "metabase/ui"; + +import { + BrowseContainer, + BrowseHeader, + BrowseMain, + BrowseSection, +} from "../components/BrowseContainer.styled"; + +import { TransformsTable } from "./TransformsTable"; + +export function BrowseTransforms() { + const { data: transforms, error, isLoading } = useListTransformsQuery(); + + return ( + + + + + + <Group gap="sm"> + <Icon + size={24} + color="var(--mb-color-icon-primary)" + name="function" + /> + {t`Transforms`} + </Group> + + + + + + + + } + > + + + + + + + ); +} diff --git a/frontend/src/metabase/browse/transforms/TransformsTable.tsx b/frontend/src/metabase/browse/transforms/TransformsTable.tsx new file mode 100644 index 000000000000..7abae3f8b53c --- /dev/null +++ b/frontend/src/metabase/browse/transforms/TransformsTable.tsx @@ -0,0 +1,213 @@ +import { type MouseEvent, useMemo } from "react"; +import { t } from "ttag"; + +import { + useDeleteTransformMutation, + useExecuteTransformMutation, +} from "metabase/api"; +import EntityItem from "metabase/common/components/EntityItem"; +import { SortableColumnHeader } from "metabase/common/components/ItemsTable/BaseItemsTable"; +import { + ColumnHeader, + ItemNameCell, + MaybeItemLink, + TBody, + Table, + TableColumn, +} from "metabase/common/components/ItemsTable/BaseItemsTable.styled"; +import { Columns } from "metabase/common/components/ItemsTable/Columns"; +import { + Button, + Icon, + type IconName, + Menu, + Repeat, + Skeleton, +} from "metabase/ui"; +import type { Transform } from "metabase-types/api"; + +import { Cell, NameColumn, TableRow } from "../components/BrowseTable.styled"; + +type TransformsTableProps = { + transforms?: Transform[]; + skeleton?: boolean; +}; + +export const itemsTableContainerName = "ItemsTableContainer"; + +const sharedProps = { + containerName: itemsTableContainerName, +}; + +const nameProps = { + ...sharedProps, +}; + +const menuProps = { + ...sharedProps, +}; + +const DOTMENU_WIDTH = 34; + +export function TransformsTable({ + transforms = [], + skeleton = false, +}: TransformsTableProps) { + return ( + + + + + + + + + + {t`Name`} + + + + + + + {skeleton ? ( + + + + ) : ( + transforms.map((transform: Transform) => ( + + )) + )} + +
+ ); +} + +function TransformRow({ transform }: { transform?: Transform }) { + return ( + + + + + + ); +} + +function SkeletonText() { + return ; +} + +function stopPropagation(event: MouseEvent) { + event.stopPropagation(); +} + +function preventDefault(event: MouseEvent) { + event.preventDefault(); +} + +function NameCell({ transform }: { transform?: Transform }) { + const headingId = `transform-${transform?.id ?? "dummy"}-heading`; + + return ( + + + paddingInlineStart: "1.4rem", + paddingInlineEnd: ".5rem", + }} + onClick={preventDefault} + > + {transform ? ( + + ) : ( + + )} + + + ); +} + +type TransformAction = { + key: string; + title: string; + icon: IconName; + action: () => void; +}; + +function MenuCell({ transform }: { transform?: Transform }) { + const [executeTransformMutation] = useExecuteTransformMutation(); + const [deleteTransformMutation] = useDeleteTransformMutation(); + + const actions = useMemo(() => { + if (!transform) { + return []; + } + + const actions: TransformAction[] = []; + actions.push({ + key: "execute", + title: t`Execute`, + icon: "play", + action: () => executeTransformMutation(transform.id), + }); + actions.push({ + key: "delete", + title: t`Delete`, + icon: "trash", + action: () => deleteTransformMutation(transform.id), + }); + + return actions; + }, [transform, executeTransformMutation, deleteTransformMutation]); + + return ( + + + + + + + {actions.map((action) => ( + } + onClick={action.action} + > + {action.title} + + ))} + + + + ); +} diff --git a/frontend/src/metabase/browse/transforms/index.ts b/frontend/src/metabase/browse/transforms/index.ts new file mode 100644 index 000000000000..3adcfe7e9e60 --- /dev/null +++ b/frontend/src/metabase/browse/transforms/index.ts @@ -0,0 +1 @@ +export * from "./BrowseTransforms"; diff --git a/frontend/src/metabase/browse/transforms/types.ts b/frontend/src/metabase/browse/transforms/types.ts new file mode 100644 index 000000000000..777e5a5dde09 --- /dev/null +++ b/frontend/src/metabase/browse/transforms/types.ts @@ -0,0 +1 @@ +export type SortColumn = "name"; diff --git a/frontend/src/metabase/nav/containers/MainNavbar/MainNavbarContainer/BrowseNavSection.tsx b/frontend/src/metabase/nav/containers/MainNavbar/MainNavbarContainer/BrowseNavSection.tsx index 4da87a3ecba4..e67cc3df8819 100644 --- a/frontend/src/metabase/nav/containers/MainNavbar/MainNavbarContainer/BrowseNavSection.tsx +++ b/frontend/src/metabase/nav/containers/MainNavbar/MainNavbarContainer/BrowseNavSection.tsx @@ -33,6 +33,7 @@ export const BrowseNavSection = ({ const BROWSE_MODELS_URL = "/browse/models"; const BROWSE_DATA_URL = "/browse/databases"; const BROWSE_METRICS_URL = "/browse/metrics"; + const BROWSE_TRANSFORMS_URL = "/browse/transforms"; const [expandBrowse = true, setExpandBrowse] = useUserSetting( "expand-browse-in-nav", @@ -123,6 +124,18 @@ export const BrowseNavSection = ({ {t`Metrics`} )} + + {!isEmbeddingIframe && ( + + {t`Transforms`} + + )} ); diff --git a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx index 27f41fc61843..2c8c2308714e 100644 --- a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx +++ b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx @@ -156,7 +156,7 @@ export const QuestionMoreActionsMenu = ({ hasDataPermissions && ( } + leftSection={} data-testid={TRANSFORM_TESTID} onClick={() => onOpenModal(MODAL_TYPES.NEW_TRANSFORM)} > diff --git a/frontend/src/metabase/routes.jsx b/frontend/src/metabase/routes.jsx index 44a28eb56a59..6869c4dc4090 100644 --- a/frontend/src/metabase/routes.jsx +++ b/frontend/src/metabase/routes.jsx @@ -16,6 +16,7 @@ import { BrowseSchemas, BrowseTables, } from "metabase/browse"; +import { BrowseTransforms } from "metabase/browse/transforms"; import { ArchiveCollectionModal } from "metabase/collections/components/ArchiveCollectionModal"; import CollectionLanding from "metabase/collections/components/CollectionLanding"; import { MoveCollectionModal } from "metabase/collections/components/MoveCollectionModal"; @@ -299,6 +300,7 @@ export const getRoutes = (store) => { + Date: Mon, 21 Jul 2025 22:27:33 +0300 Subject: [PATCH 009/164] Return an entry from the list endpoint --- .clj-kondo/config/modules/config.edn | 4 +- .../src/metabase_enterprise/api/routes.clj | 2 + .../metabase_enterprise/transforms/api.clj | 46 ++++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn index dc5e00c816af..22afcf29afb8 100644 --- a/.clj-kondo/config/modules/config.edn +++ b/.clj-kondo/config/modules/config.edn @@ -1834,7 +1834,9 @@ {:team "Querying" :api #{metabase-enterprise.transforms.api metabase-enterprise.transforms.init} - :uses #{}} + :uses #{api + query-processor + util}} enterprise/upload-management {:team "Admin Webapp" diff --git a/enterprise/backend/src/metabase_enterprise/api/routes.clj b/enterprise/backend/src/metabase_enterprise/api/routes.clj index 22693c1c18c5..5a43bce72253 100644 --- a/enterprise/backend/src/metabase_enterprise/api/routes.clj +++ b/enterprise/backend/src/metabase_enterprise/api/routes.clj @@ -27,6 +27,7 @@ [metabase-enterprise.scim.routes] [metabase-enterprise.serialization.api] [metabase-enterprise.stale.api] + [metabase-enterprise.transforms.api] [metabase-enterprise.upload-management.api] [metabase.api.macros :as api.macros] [metabase.api.util.handlers :as handlers] @@ -97,6 +98,7 @@ "/scim" (premium-handler metabase-enterprise.scim.routes/routes :scim) "/serialization" (premium-handler metabase-enterprise.serialization.api/routes :serialization) "/stale" (premium-handler metabase-enterprise.stale.api/routes :collection-cleanup) + "/transform" metabase-enterprise.transforms.api/routes "/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)}) ;;; ↑↑↑ KEEP THIS SORTED OR ELSE ↑↑↑ diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index a24c839cbfc5..c0f5366e9f24 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -1 +1,45 @@ -(ns metabase-enterprise.transforms.api) +(ns metabase-enterprise.transforms.api + (:require + [metabase.api.macros :as api.macros] + [metabase.api.routes.common :refer [+auth]] + [metabase.api.util.handlers :as handlers] + [metabase.util.malli.registry :as mr])) + +(mr/def ::transform-source + [:map + [:type [:= "query"] + :query [:map [:database :int]]]]) + +(mr/def ::transform-target + [:map + [:type [:= "table"]] + [:database :int] + [:schema {:optional true} :string] + [:table :string]]) + +(api.macros/defendpoint :get "/" + "Get a list of transforms." + [_route-params + _query-params] + [{:id 1 + :name "Gadget Products" + :source {:type "query" + :query {:database 1 + :type "native", + :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" + :template-tags {}}}} + :target {:type "table" + :database 1 + :schema "transforms" + :table "gadget_products"}}]) + +(api.macros/defendpoint :post "/" + [_route-params + _query-params + {:keys [_name _source _target] :as _body}] + _body) + +(def ^{:arglists '([request respond raise])} routes + "`/api/ee/transform` routes." + (handlers/routes + (api.macros/ns-handler *ns* +auth))) From fa0c36e0c856140d04fabc64df41723252a20fcb Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Mon, 21 Jul 2025 15:29:44 -0400 Subject: [PATCH 010/164] Transforms page --- frontend/src/metabase/api/transform.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/metabase/api/transform.ts b/frontend/src/metabase/api/transform.ts index d9e39bc6effe..55b4558911d0 100644 --- a/frontend/src/metabase/api/transform.ts +++ b/frontend/src/metabase/api/transform.ts @@ -19,7 +19,7 @@ export const transformApi = Api.injectEndpoints({ listTransforms: builder.query({ query: (params) => ({ method: "GET", - url: "/api/transform", + url: "/api/ee/transform", params, }), providesTags: (transforms = []) => provideTransformListTags(transforms), @@ -27,7 +27,7 @@ export const transformApi = Api.injectEndpoints({ getTransform: builder.query({ query: (id) => ({ method: "GET", - url: `/api/transform/${id}`, + url: `/api/ee/transform/${id}`, }), providesTags: (transform) => transform ? provideTransformTags(transform) : [], @@ -35,7 +35,7 @@ export const transformApi = Api.injectEndpoints({ createTransform: builder.mutation({ query: (body) => ({ method: "POST", - url: "/api/transform", + url: "/api/ee/transform", body, }), invalidatesTags: (_, error) => @@ -44,7 +44,7 @@ export const transformApi = Api.injectEndpoints({ executeTransform: builder.mutation({ query: (id) => ({ method: "POST", - url: `/api/transform/${id}/execute`, + url: `/api/ee/transform/${id}/execute`, }), invalidatesTags: (_, error) => invalidateTags(error, [ @@ -56,7 +56,7 @@ export const transformApi = Api.injectEndpoints({ deleteTransform: builder.mutation({ query: (id) => ({ method: "DELETE", - url: `/api/transform/${id}`, + url: `/api/ee/transform/${id}`, }), invalidatesTags: (_, error, id) => invalidateTags(error, [listTag("transform"), idTag("transform", id)]), From 29ebc2e48bd68aacaaf9632dd432398d63723b7f Mon Sep 17 00:00:00 2001 From: William Zimrin Date: Mon, 21 Jul 2025 14:31:35 -0500 Subject: [PATCH 011/164] basic transform execution --- .../transforms/execute.clj | 20 +++++++++++++++++-- src/metabase/driver.clj | 6 ++++++ src/metabase/driver/sql.clj | 3 ++- src/metabase/driver/sql/query_processor.clj | 15 ++++++++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj index 701c480e2ec6..b7f05a8642e3 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj @@ -1,4 +1,20 @@ -(ns metabase-enterprise.transforms.execute) +(ns metabase-enterprise.transforms.execute + (:require [metabase.driver :as driver] + [metabase.query-processor.preprocess :as qp.preprocess] + [metabase.query-processor.setup :as qp.setup])) + +(defn- execute-query [driver db sql] + (let [query {:native {:query sql} + :type :native + :database db}] + (qp.setup/with-qp-setup [query query] + (let [query (qp.preprocess/preprocess query)] + (driver/execute-write-query! driver query))))) (defn execute [{:keys [db driver sql output-table overwrite?]}] - (or output-table (str "transform_" (random-uuid)))) + (let [output-table (keyword (or output-table (str "transform_" (random-uuid)))) + query (driver/compile-transform driver {:sql sql :output-table output-table :overwrite? overwrite?})] + (when overwrite? + (execute-query driver db (driver/drop-transform driver output-table))) + (execute-query driver db query) + output-table)) diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj index 2c19088ecdc6..78b3afbd939f 100644 --- a/src/metabase/driver.clj +++ b/src/metabase/driver.clj @@ -702,6 +702,8 @@ ;; Does this driver support "temporal-unit" template tags in native queries? :native-temporal-units + :transforms/basic + ;; Whether the driver supports loading dynamic test datasets on each test run. Eg. datasets with names like ;; `checkins:4-per-minute` are created dynamically in each test run. This should be truthy for every driver we test ;; against except for Athena and Databricks which currently require test data to be loaded separately. @@ -1351,3 +1353,7 @@ :hierarchy #'hierarchy) (defmethod table-known-to-not-exist? ::driver [_ _] false) + +(defmulti compile-transform dispatch-on-initialized-driver :hierarchy #'hierarchy) + +(defmulti drop-transform dispatch-on-initialized-driver :hierarchy #'hierarchy) diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj index 78a8ee5197a4..1ed2b076a9c9 100644 --- a/src/metabase/driver/sql.clj +++ b/src/metabase/driver/sql.clj @@ -41,7 +41,8 @@ :expressions/text :expressions/today :distinct-where - :database-routing]] + :database-routing + :transforms/basic]] (defmethod driver/database-supports? [:sql feature] [_driver _feature _db] true)) (defmethod driver/database-supports? [:sql :persist-models-enabled] diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj index d884b3713173..4b04ec44346c 100644 --- a/src/metabase/driver/sql/query_processor.clj +++ b/src/metabase/driver/sql/query_processor.clj @@ -2070,3 +2070,18 @@ (let [honeysql-form (mbql->honeysql driver outer-query) [sql & args] (format-honeysql driver honeysql-form)] {:query sql, :params args})) + +;;;; Transforms + +(defmethod driver/compile-transform :sql + [driver {:keys [sql output-table]}] + (first + (format-honeysql driver + {:select [:*] + :from [[[:raw (str "(" sql ")")] (str (random-uuid))]] + :into output-table}))) + +(defmethod driver/drop-transform :sql + [driver table] + (first + (format-honeysql driver {:drop-table [:if-exists table]}))) From 7620abe8f76058e9081c45bbe6ba3d4eaff15a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Mon, 21 Jul 2025 22:32:56 +0300 Subject: [PATCH 012/164] Fix POST input schema --- .../backend/src/metabase_enterprise/transforms/api.clj | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index c0f5366e9f24..3b3835d82308 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -7,8 +7,8 @@ (mr/def ::transform-source [:map - [:type [:= "query"] - :query [:map [:database :int]]]]) + [:type [:= "query"]] + [:query [:map [:database :int]]]]) (mr/def ::transform-target [:map @@ -36,7 +36,10 @@ (api.macros/defendpoint :post "/" [_route-params _query-params - {:keys [_name _source _target] :as _body}] + {:keys [_name _source _target] :as _body} :- [:map + [:name :string] + [:source ::transform-source] + [:target ::transform-target]]] _body) (def ^{:arglists '([request respond raise])} routes From 7a9c5ba1eeab757a12ef3a1279f2e027666f4f4c Mon Sep 17 00:00:00 2001 From: William Zimrin Date: Mon, 21 Jul 2025 15:00:48 -0500 Subject: [PATCH 013/164] some untested dummy routes --- .../metabase_enterprise/transforms/api.clj | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 3b3835d82308..70156a1e72c2 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -1,9 +1,12 @@ (ns metabase-enterprise.transforms.api (:require + [metabase-enterprise.transforms.execute :as transforms.execute] [metabase.api.macros :as api.macros] [metabase.api.routes.common :refer [+auth]] [metabase.api.util.handlers :as handlers] - [metabase.util.malli.registry :as mr])) + [metabase.query-processor.compile :as qp.compile] + [metabase.util.malli.registry :as mr] + [toucan2.core :as t2])) (mr/def ::transform-source [:map @@ -42,6 +45,61 @@ [:target ::transform-target]]] _body) +(api.macros/defendpoint :get "/:id" + [{:keys [id]}] + (prn "get transform" id) + {:id 1 + :name "Gadget Products" + :source {:type "query" + :query {:database 1 + :type "native", + :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" + :template-tags {}}}} + :target {:type "table" + :database 1 + :schema "transforms" + :table "gadget_products"}}) + +(api.macros/defendpoint :put "/:id" + [{:keys [id]} + _query-params + {:keys [_name _source _target] :as _body} :- [:map + [:name :string] + [:source ::transform-source] + [:target ::transform-target]]] + (prn "put transform" id) + _body) + +(api.macros/defendpoint :delete "/:id" + [{:keys [id]}] + (prn "delete transform" id) + "success") + +(api.macros/defendpoint :post "/:id/execute" + [{:keys [id]}] + (prn "execute transform" id) + (let [{:keys [name source target]} + {:id 1 + :name "Gadget Products" + :source {:type "query" + :query {:database 1 + :type "native", + :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" + :template-tags {}}}} + :target {:type "table" + :database 1 + :schema "transforms" + :table "gadget_products"}} + + db (get-in source [:query :database]) + {driver :engine} (t2/select-one :model/Database db)] + (transforms.execute/execute + {:db db + :driver driver + :sql (first (qp.compile/compile source)) + :output-table (:table target) + :overwrite? true}))) + (def ^{:arglists '([request respond raise])} routes "`/api/ee/transform` routes." (handlers/routes From 717b6bda5ac3460334c3dfc7a3b98ec240f7300e Mon Sep 17 00:00:00 2001 From: William Zimrin Date: Mon, 21 Jul 2025 15:02:31 -0500 Subject: [PATCH 014/164] use compile-with-inline-parameters --- enterprise/backend/src/metabase_enterprise/transforms/api.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 70156a1e72c2..12262738ec78 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -96,7 +96,7 @@ (transforms.execute/execute {:db db :driver driver - :sql (first (qp.compile/compile source)) + :sql (first (qp.compile/compile-with-inline-parameters source)) :output-table (:table target) :overwrite? true}))) From d37fdca122589beaa3acff4f434b1a0c13d33b59 Mon Sep 17 00:00:00 2001 From: William Zimrin Date: Mon, 21 Jul 2025 15:03:48 -0500 Subject: [PATCH 015/164] actually fetch the query from the result of compile --- enterprise/backend/src/metabase_enterprise/transforms/api.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 12262738ec78..05b1fb2577ad 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -96,7 +96,7 @@ (transforms.execute/execute {:db db :driver driver - :sql (first (qp.compile/compile-with-inline-parameters source)) + :sql (:query (qp.compile/compile-with-inline-parameters source)) :output-table (:table target) :overwrite? true}))) From ec44b108ffa582028f9661415ea04b9c6ac5e98d Mon Sep 17 00:00:00 2001 From: William Zimrin Date: Mon, 21 Jul 2025 15:16:28 -0500 Subject: [PATCH 016/164] make execute actually execute --- .../backend/src/metabase_enterprise/transforms/api.clj | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 05b1fb2577ad..20e5dc6a3ddb 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -75,6 +75,10 @@ (prn "delete transform" id) "success") +(defn- compile-source [{query-type :type :as source}] + (case query-type + "query" (:query (qp.compile/compile-with-inline-parameters (:query source))))) + (api.macros/defendpoint :post "/:id/execute" [{:keys [id]}] (prn "execute transform" id) @@ -96,7 +100,7 @@ (transforms.execute/execute {:db db :driver driver - :sql (:query (qp.compile/compile-with-inline-parameters source)) + :sql (compile-source source) :output-table (:table target) :overwrite? true}))) From eb4e329cd26e6f8193d0362cf501e2ac48d1d469 Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Mon, 21 Jul 2025 15:19:19 -0500 Subject: [PATCH 017/164] Transform migration + toucan model --- .clj-kondo/config/modules/config.edn | 1 + .../transforms/models/transform.clj | 17 ++++- .../migrations/056_update_migrations.yaml | 63 +++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn index 22afcf29afb8..61ae57f2a811 100644 --- a/.clj-kondo/config/modules/config.edn +++ b/.clj-kondo/config/modules/config.edn @@ -1835,6 +1835,7 @@ :api #{metabase-enterprise.transforms.api metabase-enterprise.transforms.init} :uses #{api + models query-processor util}} diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj index 44551161e9f4..e1a704757f70 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj @@ -1 +1,16 @@ -(ns metabase-enterprise.transforms.models.transform) +(ns metabase-enterprise.transforms.models.transform + (:require + [metabase.models.interface :as mi] + [methodical.core :as methodical] + [toucan2.core :as t2])) + +(methodical/defmethod t2/table-name :model/Transform [_model] :transforms) + +(doto :model/Transform + (derive :metabase/model) + (derive :hook/entity-id) + (derive :hook/timestamped?)) + +(t2/deftransforms :model/Transform + {:source mi/transform-json + :target mi/transform-json}) diff --git a/resources/migrations/056_update_migrations.yaml b/resources/migrations/056_update_migrations.yaml index 952cb15e15f1..3956ddde72aa 100644 --- a/resources/migrations/056_update_migrations.yaml +++ b/resources/migrations/056_update_migrations.yaml @@ -201,6 +201,69 @@ databaseChangeLog: path: instance_analytics_views/users/v2/h2-users.sql relativeToChangelogFile: true + - changeSet: + id: v56.2025-07-21T20:40:00 + author: ericnormand + comment: Create transforms table + preConditions: + - onFail: MARK_RAN + - not: + - tableExists: + tableName: transforms + changes: + - createTable: + tableName: transforms + remarks: TODO + columns: + - column: + name: id + remarks: Unique ID + type: int + autoIncrement: true + constraints: + primaryKey: true + nullable: false + - column: + name: name + remarks: Name + type: ${text.type} + constraints: + nullable: false + - column: + name: source + remarks: JSON of source + type: ${text.type} + constraints: + nullable: false + - column: + name: target + remarks: JSON of target + type: ${text.type} + constraints: + nullable: false + - column: + name: entity_id + type: char(21) + remarks: Random NanoID tag for unique identity. + constraints: + nullable: false + unique: true + - column: + remarks: The timestamp of when the transform was created + name: created_at + type: ${timestamp_type} + defaultValueComputed: current_timestamp + constraints: + nullable: false + - column: + remarks: The timestamp of when the transform was updated + name: updated_at + type: ${timestamp_type} + defaultValueComputed: current_timestamp + constraints: + nullable: false + + # >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<< ######################################################################################################################## From f276dabad95867dc348b74d80032d30055dac422 Mon Sep 17 00:00:00 2001 From: William Zimrin Date: Mon, 21 Jul 2025 15:31:42 -0500 Subject: [PATCH 018/164] h2 doesn't support transforms --- src/metabase/driver/h2.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.clj index 0b6e5dad10f2..d37dc6562445 100644 --- a/src/metabase/driver/h2.clj +++ b/src/metabase/driver/h2.clj @@ -69,7 +69,8 @@ :test/jvm-timezone-setting false :uuid-type true :uploads true - :database-routing true}] + :database-routing true + :transforms/basic false}] (defmethod driver/database-supports? [:h2 feature] [_driver _feature _database] supported?)) From 9320a8fcc213242834e8a5d55f1f103f9b7cb642 Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Mon, 21 Jul 2025 15:44:19 -0500 Subject: [PATCH 019/164] Use toucan to implement API endpoints --- .../metabase_enterprise/transforms/api.clj | 64 ++++++++----------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 20e5dc6a3ddb..a8d20cc0a723 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -20,10 +20,8 @@ [:schema {:optional true} :string] [:table :string]]) -(api.macros/defendpoint :get "/" - "Get a list of transforms." - [_route-params - _query-params] +(comment + ;; Examples [{:id 1 :name "Gadget Products" :source {:type "query" @@ -36,44 +34,44 @@ :schema "transforms" :table "gadget_products"}}]) +(api.macros/defendpoint :get "/" + "Get a list of transforms." + [_route-params + _query-params] + (t2/select :model/Transform)) + (api.macros/defendpoint :post "/" [_route-params _query-params - {:keys [_name _source _target] :as _body} :- [:map - [:name :string] - [:source ::transform-source] - [:target ::transform-target]]] - _body) + {:keys [name source target] :as _body} :- [:map + [:name :string] + [:source ::transform-source] + [:target ::transform-target]]] + (t2/insert-returning-pk! :model/Transform {:name name + :source source + :target target})) (api.macros/defendpoint :get "/:id" [{:keys [id]}] (prn "get transform" id) - {:id 1 - :name "Gadget Products" - :source {:type "query" - :query {:database 1 - :type "native", - :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" - :template-tags {}}}} - :target {:type "table" - :database 1 - :schema "transforms" - :table "gadget_products"}}) + (t2/select-one :model/Transform id)) (api.macros/defendpoint :put "/:id" [{:keys [id]} _query-params - {:keys [_name _source _target] :as _body} :- [:map - [:name :string] - [:source ::transform-source] - [:target ::transform-target]]] + {:keys [name source target] :as _body} :- [:map + [:name :string] + [:source ::transform-source] + [:target ::transform-target]]] (prn "put transform" id) - _body) + (t2/update! :model/Transform id {:name name + :source source + :target target})) (api.macros/defendpoint :delete "/:id" [{:keys [id]}] (prn "delete transform" id) - "success") + (t2/delete! :model/Transform id)) (defn- compile-source [{query-type :type :as source}] (case query-type @@ -82,19 +80,7 @@ (api.macros/defendpoint :post "/:id/execute" [{:keys [id]}] (prn "execute transform" id) - (let [{:keys [name source target]} - {:id 1 - :name "Gadget Products" - :source {:type "query" - :query {:database 1 - :type "native", - :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" - :template-tags {}}}} - :target {:type "table" - :database 1 - :schema "transforms" - :table "gadget_products"}} - + (let [{:keys [_name source target]} (t2/select-one :model/Transform id) db (get-in source [:query :database]) {driver :engine} (t2/select-one :model/Database db)] (transforms.execute/execute From fccfa9460b4c5f3c3407646f52a35fb15d3800ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 00:04:58 +0300 Subject: [PATCH 020/164] Placate clj-kondo --- .clj-kondo/config/modules/config.edn | 1 + .../src/metabase_enterprise/transforms/api.clj | 16 +++++++++------- .../metabase_enterprise/transforms/execute.clj | 7 ++++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn index 61ae57f2a811..eca6af133b2d 100644 --- a/.clj-kondo/config/modules/config.edn +++ b/.clj-kondo/config/modules/config.edn @@ -1835,6 +1835,7 @@ :api #{metabase-enterprise.transforms.api metabase-enterprise.transforms.init} :uses #{api + driver models query-processor util}} diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index a8d20cc0a723..4af74736f1df 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -5,6 +5,7 @@ [metabase.api.routes.common :refer [+auth]] [metabase.api.util.handlers :as handlers] [metabase.query-processor.compile :as qp.compile] + [metabase.util.log :as log] [metabase.util.malli.registry :as mr] [toucan2.core :as t2])) @@ -47,13 +48,14 @@ [:name :string] [:source ::transform-source] [:target ::transform-target]]] - (t2/insert-returning-pk! :model/Transform {:name name - :source source - :target target})) + (let [id (t2/insert-returning-pk! :model/Transform {:name name + :source source + :target target})] + (t2/select-one :model/Transform id))) (api.macros/defendpoint :get "/:id" [{:keys [id]}] - (prn "get transform" id) + (log/info "get transform" id) (t2/select-one :model/Transform id)) (api.macros/defendpoint :put "/:id" @@ -63,14 +65,14 @@ [:name :string] [:source ::transform-source] [:target ::transform-target]]] - (prn "put transform" id) + (log/info "put transform" id) (t2/update! :model/Transform id {:name name :source source :target target})) (api.macros/defendpoint :delete "/:id" [{:keys [id]}] - (prn "delete transform" id) + (log/info "delete transform" id) (t2/delete! :model/Transform id)) (defn- compile-source [{query-type :type :as source}] @@ -79,7 +81,7 @@ (api.macros/defendpoint :post "/:id/execute" [{:keys [id]}] - (prn "execute transform" id) + (log/info "execute transform" id) (let [{:keys [_name source target]} (t2/select-one :model/Transform id) db (get-in source [:query :database]) {driver :engine} (t2/select-one :model/Database db)] diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj index b7f05a8642e3..5bf3e09c45f9 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj @@ -1,7 +1,8 @@ (ns metabase-enterprise.transforms.execute - (:require [metabase.driver :as driver] - [metabase.query-processor.preprocess :as qp.preprocess] - [metabase.query-processor.setup :as qp.setup])) + (:require + [metabase.driver :as driver] + [metabase.query-processor.preprocess :as qp.preprocess] + [metabase.query-processor.setup :as qp.setup])) (defn- execute-query [driver db sql] (let [query {:native {:query sql} From 27edaf23e6e8968ad4e48c19a52901caf46b6f47 Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Mon, 21 Jul 2025 16:46:39 -0500 Subject: [PATCH 021/164] Delete table. --- .../src/metabase_enterprise/transforms/api.clj | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 4af74736f1df..349b10bcbb64 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -4,6 +4,7 @@ [metabase.api.macros :as api.macros] [metabase.api.routes.common :refer [+auth]] [metabase.api.util.handlers :as handlers] + [metabase.driver :as driver] [metabase.query-processor.compile :as qp.compile] [metabase.util.log :as log] [metabase.util.malli.registry :as mr] @@ -70,11 +71,23 @@ :source source :target target})) +(defn- delete-target-table! [id] + (let [{:keys [_name _source target]} (t2/select-one :model/Transform id) + {:keys [database table]} target + {driver :engine} (t2/select-one :model/Database database)] + (driver/drop-table! driver database table))) + (api.macros/defendpoint :delete "/:id" [{:keys [id]}] (log/info "delete transform" id) + (delete-target-table! id) (t2/delete! :model/Transform id)) +(api.macros/defendpoint :delete "/:id/table" + [{:keys [id]}] + (log/info "delete transform target table" id) + (delete-target-table! id)) + (defn- compile-source [{query-type :type :as source}] (case query-type "query" (:query (qp.compile/compile-with-inline-parameters (:query source))))) From 9b8f61def1040498a7ce039fe71dd93517381abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 00:53:07 +0300 Subject: [PATCH 022/164] Check native query execution right when executing a transform --- .clj-kondo/config/modules/config.edn | 1 + .../backend/src/metabase_enterprise/transforms/api.clj | 5 +++++ .../backend/src/metabase_enterprise/transforms/execute.clj | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn index eca6af133b2d..2c5034fc44c4 100644 --- a/.clj-kondo/config/modules/config.edn +++ b/.clj-kondo/config/modules/config.edn @@ -1837,6 +1837,7 @@ :uses #{api driver models + permissions query-processor util}} diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 349b10bcbb64..46ef1c775cfa 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -1,10 +1,12 @@ (ns metabase-enterprise.transforms.api (:require [metabase-enterprise.transforms.execute :as transforms.execute] + [metabase.api.common :as api] [metabase.api.macros :as api.macros] [metabase.api.routes.common :refer [+auth]] [metabase.api.util.handlers :as handlers] [metabase.driver :as driver] + [metabase.permissions.core :as perms] [metabase.query-processor.compile :as qp.compile] [metabase.util.log :as log] [metabase.util.malli.registry :as mr] @@ -98,6 +100,9 @@ (let [{:keys [_name source target]} (t2/select-one :model/Transform id) db (get-in source [:query :database]) {driver :engine} (t2/select-one :model/Database db)] + (when (not= (perms/full-db-permission-for-user api/*current-user-id* :perms/create-queries db) + :query-builder-and-native) + (api/throw-403)) (transforms.execute/execute {:db db :driver driver diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj index 5bf3e09c45f9..4a4a7e316501 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj @@ -1,5 +1,6 @@ (ns metabase-enterprise.transforms.execute (:require + [clojure.string :as str] [metabase.driver :as driver] [metabase.query-processor.preprocess :as qp.preprocess] [metabase.query-processor.setup :as qp.setup])) @@ -13,7 +14,7 @@ (driver/execute-write-query! driver query))))) (defn execute [{:keys [db driver sql output-table overwrite?]}] - (let [output-table (keyword (or output-table (str "transform_" (random-uuid)))) + (let [output-table (keyword (or output-table (str "transform_" (str/replace (random-uuid) \- \_)))) query (driver/compile-transform driver {:sql sql :output-table output-table :overwrite? overwrite?})] (when overwrite? (execute-query driver db (driver/drop-transform driver output-table))) From a8d1dcc496a077ebdd3d2db3233a21df1e0400b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 01:27:06 +0300 Subject: [PATCH 023/164] [ci skip] Enable selective updates --- .../metabase_enterprise/transforms/api.clj | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 46ef1c775cfa..52edeed9e720 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -64,26 +64,25 @@ (api.macros/defendpoint :put "/:id" [{:keys [id]} _query-params - {:keys [name source target] :as _body} :- [:map - [:name :string] - [:source ::transform-source] - [:target ::transform-target]]] + body :- [:map + [:name {:optional true} :string] + [:source {:optional true} ::transform-source] + [:target {:optional true} ::transform-target]]] (log/info "put transform" id) - (t2/update! :model/Transform id {:name name - :source source - :target target})) + (t2/update! :model/Transform id (select-keys body [:name :source :target])) + (t2/select-one :model/Transform id)) (defn- delete-target-table! [id] - (let [{:keys [_name _source target]} (t2/select-one :model/Transform id) - {:keys [database table]} target - {driver :engine} (t2/select-one :model/Database database)] + (let [{:keys [database table]} (t2/select-one-fn :target :model/Transform id) + driver (t2/select-one-fn :engine :model/Database database)] (driver/drop-table! driver database table))) (api.macros/defendpoint :delete "/:id" [{:keys [id]}] (log/info "delete transform" id) (delete-target-table! id) - (t2/delete! :model/Transform id)) + (t2/delete! :model/Transform id) + nil) (api.macros/defendpoint :delete "/:id/table" [{:keys [id]}] From 87c5dd88d15e9d613b3fc00172dff7cc8f073a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 01:37:40 +0300 Subject: [PATCH 024/164] [ci skip] Prepend schema to table if provided --- .../backend/src/metabase_enterprise/transforms/api.clj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 52edeed9e720..31578bae384d 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -36,7 +36,8 @@ :target {:type "table" :database 1 :schema "transforms" - :table "gadget_products"}}]) + :table "gadget_products"}}] + -) (api.macros/defendpoint :get "/" "Get a list of transforms." @@ -106,7 +107,8 @@ {:db db :driver driver :sql (compile-source source) - :output-table (:table target) + :output-table (cond->> (:table target) + (:schema target) (str (:schema target) ".")) :overwrite? true}))) (def ^{:arglists '([request respond raise])} routes From 0648c78e1a1ce4e7200cb47f2cb8e16f8bb1c4fe Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Mon, 21 Jul 2025 20:38:35 -0400 Subject: [PATCH 025/164] Transforms page --- .../QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx index 2c8c2308714e..6ac14332f608 100644 --- a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx +++ b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionActions/QuestionMoreActionsMenu/QuestionMoreActionsMenu.tsx @@ -160,7 +160,7 @@ export const QuestionMoreActionsMenu = ({ data-testid={TRANSFORM_TESTID} onClick={() => onOpenModal(MODAL_TYPES.NEW_TRANSFORM)} > - {t`Turn into a transform`} + {t`Create a transform`} ), hasCollectionPermissions && isModel && ( From aaee0896e6c3fa0ab775a1aae8444e0f5a6426f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 15:11:55 +0300 Subject: [PATCH 026/164] Require metabase-enterprise.transforms.models.transform in init --- enterprise/backend/src/metabase_enterprise/transforms/init.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/init.clj b/enterprise/backend/src/metabase_enterprise/transforms/init.clj index e1517a954cbd..2aeab4d20d07 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/init.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/init.clj @@ -1,3 +1,3 @@ (ns metabase-enterprise.transforms.init (:require - [metabase-enterprise.metabot-v3.settings])) + [metabase-enterprise.transforms.models.transform])) From a550e694b8697189f4b940617e37f08ba5f847fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 15:56:52 +0300 Subject: [PATCH 027/164] [skip ci] Drop target database-id, check if the target table exists --- .../metabase_enterprise/transforms/api.clj | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 31578bae384d..19c9bcf7d492 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -20,7 +20,6 @@ (mr/def ::transform-target [:map [:type [:= "table"]] - [:database :int] [:schema {:optional true} :string] [:table :string]]) @@ -39,6 +38,28 @@ :table "gadget_products"}}] -) +;; TODO add driver specific quoting +(defn- qualified-table-name + [{:keys [schema table]}] + (cond->> table + schema (str schema "."))) + +(defn- target-table-exists? + [{:keys [source target] :as _transform}] + (let [database (-> source :query :database) + driver (t2/select-one-fn :engine :model/Database database)] + (some? (driver/describe-table driver database (qualified-table-name target))))) + +(defn- delete-target-table! + [{:keys [source target] :as _transform}] + (let [database (-> source :query :database) + driver (t2/select-one-fn :engine :model/Database database)] + (driver/drop-table! driver database (qualified-table-name target)))) + +(defn- delete-target-table-by-id! + [transform-id] + (delete-target-table! (t2/select-one :model/Transform transform-id))) + (api.macros/defendpoint :get "/" "Get a list of transforms." [_route-params @@ -48,10 +69,12 @@ (api.macros/defendpoint :post "/" [_route-params _query-params - {:keys [name source target] :as _body} :- [:map - [:name :string] - [:source ::transform-source] - [:target ::transform-target]]] + {:keys [name source target] :as body} :- [:map + [:name :string] + [:source ::transform-source] + [:target ::transform-target]]] + (when (target-table-exists? body) + (api/throw-403)) (let [id (t2/insert-returning-pk! :model/Transform {:name name :source source :target target})] @@ -70,25 +93,26 @@ [:source {:optional true} ::transform-source] [:target {:optional true} ::transform-target]]] (log/info "put transform" id) - (t2/update! :model/Transform id (select-keys body [:name :source :target])) + (let [old (t2/select-one-fn :target :model/Transform id) + new (merge old body)] + (when (not= (select-keys (:target old) [:schema :table]) + (select-keys (:target new) [:schema :table])) + (when (target-table-exists? new) + (api/throw-403)) + (delete-target-table! old))) (t2/select-one :model/Transform id)) -(defn- delete-target-table! [id] - (let [{:keys [database table]} (t2/select-one-fn :target :model/Transform id) - driver (t2/select-one-fn :engine :model/Database database)] - (driver/drop-table! driver database table))) - (api.macros/defendpoint :delete "/:id" [{:keys [id]}] (log/info "delete transform" id) - (delete-target-table! id) + (delete-target-table-by-id! id) (t2/delete! :model/Transform id) nil) (api.macros/defendpoint :delete "/:id/table" [{:keys [id]}] (log/info "delete transform target table" id) - (delete-target-table! id)) + (delete-target-table-by-id! id)) (defn- compile-source [{query-type :type :as source}] (case query-type @@ -107,8 +131,7 @@ {:db db :driver driver :sql (compile-source source) - :output-table (cond->> (:table target) - (:schema target) (str (:schema target) ".")) + :output-table (qualified-table-name target) :overwrite? true}))) (def ^{:arglists '([request respond raise])} routes From b7247b23e88ea642e99ccbf7104f122df95ef559 Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Tue, 22 Jul 2025 10:56:27 -0500 Subject: [PATCH 028/164] [ci skip] Better api parameter handling --- .../metabase_enterprise/transforms/api.clj | 21 ++++++++++++------- .../metabase_enterprise/transforms/init.clj | 3 ++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 349b10bcbb64..7e69a1eb6773 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -10,6 +10,8 @@ [metabase.util.malli.registry :as mr] [toucan2.core :as t2])) +(set! *warn-on-reflection* true) + (mr/def ::transform-source [:map [:type [:= "query"]] @@ -57,7 +59,7 @@ (api.macros/defendpoint :get "/:id" [{:keys [id]}] (log/info "get transform" id) - (t2/select-one :model/Transform id)) + (t2/select-one :model/Transform (Long/parseLong id))) (api.macros/defendpoint :put "/:id" [{:keys [id]} @@ -67,26 +69,29 @@ [:source ::transform-source] [:target ::transform-target]]] (log/info "put transform" id) - (t2/update! :model/Transform id {:name name - :source source - :target target})) + (t2/update! :model/Transform (Long/parseLong id) {:name name + :source source + :target target})) (defn- delete-target-table! [id] + (prn (t2/select-one :model/Transform id)) (let [{:keys [_name _source target]} (t2/select-one :model/Transform id) {:keys [database table]} target - {driver :engine} (t2/select-one :model/Database database)] + _ (prn database) + {driver :engine :as poop} (t2/select-one :model/Database database)] + (prn poop) (driver/drop-table! driver database table))) (api.macros/defendpoint :delete "/:id" [{:keys [id]}] (log/info "delete transform" id) - (delete-target-table! id) - (t2/delete! :model/Transform id)) + (delete-target-table! (Long/parseLong id)) + (t2/delete! :model/Transform (Long/parseLong id))) (api.macros/defendpoint :delete "/:id/table" [{:keys [id]}] (log/info "delete transform target table" id) - (delete-target-table! id)) + (delete-target-table! (Long/parseLong id))) (defn- compile-source [{query-type :type :as source}] (case query-type diff --git a/enterprise/backend/src/metabase_enterprise/transforms/init.clj b/enterprise/backend/src/metabase_enterprise/transforms/init.clj index e1517a954cbd..9312b8acef58 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/init.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/init.clj @@ -1,3 +1,4 @@ (ns metabase-enterprise.transforms.init (:require - [metabase-enterprise.metabot-v3.settings])) + [metabase-enterprise.metabot-v3.settings] + [metabase-enterprise.transforms.models.transform])) From 38c5fa58925e675f6c0c95dea778562de9122eb9 Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Tue, 22 Jul 2025 10:57:21 -0500 Subject: [PATCH 029/164] [ci skip] more parsing --- enterprise/backend/src/metabase_enterprise/transforms/api.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 7e69a1eb6773..81aed5e58fe4 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -100,7 +100,7 @@ (api.macros/defendpoint :post "/:id/execute" [{:keys [id]}] (log/info "execute transform" id) - (let [{:keys [_name source target]} (t2/select-one :model/Transform id) + (let [{:keys [_name source target]} (t2/select-one :model/Transform (Long/parseLong id)) db (get-in source [:query :database]) {driver :engine} (t2/select-one :model/Database db)] (transforms.execute/execute From febfce82604ee70dabd183aba9e62512cd32c8ba Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 22 Jul 2025 12:00:10 -0400 Subject: [PATCH 030/164] Fix --- .../metabase_enterprise/transforms/api.clj | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 19c9bcf7d492..86683a9774e9 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -10,6 +10,7 @@ [metabase.query-processor.compile :as qp.compile] [metabase.util.log :as log] [metabase.util.malli.registry :as mr] + [metabase.util.malli.schema :as ms] [toucan2.core :as t2])) (mr/def ::transform-source @@ -46,9 +47,10 @@ (defn- target-table-exists? [{:keys [source target] :as _transform}] - (let [database (-> source :query :database) - driver (t2/select-one-fn :engine :model/Database database)] - (some? (driver/describe-table driver database (qualified-table-name target))))) + false + #_(let [database (-> source :query :database) + driver (t2/select-one-fn :engine :model/Database database)] + (some? (driver/describe-table driver database (qualified-table-name target))))) (defn- delete-target-table! [{:keys [source target] :as _transform}] @@ -81,12 +83,14 @@ (t2/select-one :model/Transform id))) (api.macros/defendpoint :get "/:id" - [{:keys [id]}] + [{:keys [id]} :- [:map + [:id ms/PositiveInt]]] (log/info "get transform" id) (t2/select-one :model/Transform id)) (api.macros/defendpoint :put "/:id" - [{:keys [id]} + [{:keys [id]} :- [:map + [:id ms/PositiveInt]] _query-params body :- [:map [:name {:optional true} :string] @@ -103,14 +107,16 @@ (t2/select-one :model/Transform id)) (api.macros/defendpoint :delete "/:id" - [{:keys [id]}] + [{:keys [id]} :- [:map + [:id ms/PositiveInt]]] (log/info "delete transform" id) - (delete-target-table-by-id! id) + #_(delete-target-table-by-id! id) (t2/delete! :model/Transform id) nil) (api.macros/defendpoint :delete "/:id/table" - [{:keys [id]}] + [{:keys [id]} :- [:map + [:id ms/PositiveInt]]] (log/info "delete transform target table" id) (delete-target-table-by-id! id)) @@ -119,7 +125,8 @@ "query" (:query (qp.compile/compile-with-inline-parameters (:query source))))) (api.macros/defendpoint :post "/:id/execute" - [{:keys [id]}] + [{:keys [id]} :- [:map + [:id ms/PositiveInt]]] (log/info "execute transform" id) (let [{:keys [_name source target]} (t2/select-one :model/Transform id) db (get-in source [:query :database]) From e64ecbbcadf6644bf4bc19e866e528f2f7c7cda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 19:04:02 +0300 Subject: [PATCH 031/164] Fix table exists check --- .../backend/src/metabase_enterprise/transforms/api.clj | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 19c9bcf7d492..aa05473ccafe 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -1,5 +1,6 @@ (ns metabase-enterprise.transforms.api (:require + [clojure.set :as set] [metabase-enterprise.transforms.execute :as transforms.execute] [metabase.api.common :as api] [metabase.api.macros :as api.macros] @@ -48,7 +49,10 @@ [{:keys [source target] :as _transform}] (let [database (-> source :query :database) driver (t2/select-one-fn :engine :model/Database database)] - (some? (driver/describe-table driver database (qualified-table-name target))))) + (-> (driver/describe-table driver database (set/rename-keys target {:table :name})) + :fields + seq + boolean))) (defn- delete-target-table! [{:keys [source target] :as _transform}] From 706d7f5dc4f2cfb2940d930b0159ecdeadbb9c11 Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 22 Jul 2025 12:07:12 -0400 Subject: [PATCH 032/164] Fix --- .../src/metabase_enterprise/transforms/api.clj | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index e7874835beb6..beca3b8ddbab 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -48,13 +48,12 @@ (defn- target-table-exists? [{:keys [source target] :as _transform}] - false - #_(let [database (-> source :query :database) - driver (t2/select-one-fn :engine :model/Database database)] - (-> (driver/describe-table driver database (set/rename-keys target {:table :name})) - :fields - seq - boolean))) + (let [database (-> source :query :database) + driver (t2/select-one-fn :engine :model/Database database)] + (-> (driver/describe-table driver database (set/rename-keys target {:table :name})) + :fields + seq + boolean))) (defn- delete-target-table! [{:keys [source target] :as _transform}] From 9967f4fa07ef865841afd06e4affc60247b64f4a Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Tue, 22 Jul 2025 11:34:30 -0500 Subject: [PATCH 033/164] [ci skip] Transforms api tests --- .../transforms/api_test.clj | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 enterprise/backend/test/metabase_enterprise/transforms/api_test.clj diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj new file mode 100644 index 000000000000..2aabed6f2de4 --- /dev/null +++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj @@ -0,0 +1,97 @@ +(ns ^:mb/driver-tests metabase-enterprise.transforms.api-test + "Tests for /api/transform endpoints." + (:require + [clojure.test :refer :all] + [metabase.test :as mt])) + +(set! *warn-on-reflection* true) + +(deftest list-transforms-test + (mt/test-drivers (mt/normal-drivers) + (mt/user-http-request :rasta :get 200 "ee/transform"))) + +(deftest create-transform-test + (mt/test-drivers (mt/normal-drivers) + (mt/user-http-request :rasta :post 200 "ee/transform" + {:name "Gadget Products" + :source {:type "query" + :query {:database (mt/id) + :type "native", + :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" + :template-tags {}}}} + :target {:type "table" + :database (mt/id) + ;; leave out schema for now + ;;:schema (str (rand-int 10000)) + :table "gadget_products"}}))) + +(deftest get-transforms-test + (mt/test-drivers (mt/normal-drivers) + (let [resp (mt/user-http-request :rasta :post 200 "ee/transform" + {:name "Gadget Products" + :source {:type "query" + :query {:database (mt/id) + :type "native", + :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" + :template-tags {}}}} + :target {:type "table" + :database (mt/id) + :schema "transforms" + :table "gadget_products"}})] + (mt/user-http-request :rasta :get 200 (format "ee/transform/%s" (:id resp)))))) + +(deftest put-transforms-test + (mt/test-drivers (mt/normal-drivers) + (let [resp (mt/user-http-request :rasta :post 200 "ee/transform" + {:name "Gadget Products" + :source {:type "query" + :query {:database (mt/id) + :type "native", + :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" + :template-tags {}}}} + :target {:type "table" + :database (mt/id) + :schema "transforms" + :table "gadget_products"}})] + (mt/user-http-request :rasta :put 200 (format "ee/transform/%s" (:id resp)) + {:name "Gadget Products 2" + :source {:type "query" + :query {:database (mt/id) + :type "native", + :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'None'" + :template-tags {}}}} + :target {:type "table" + :database (mt/id) + :schema "transforms" + :table "gadget_products"}})))) + +(deftest delete-transforms-test + (mt/test-drivers (mt/normal-drivers) + (let [resp (mt/user-http-request :rasta :post 200 "ee/transform" + {:name "Gadget Products" + :source {:type "query" + :query {:database (mt/id) + :type "native", + :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" + :template-tags {}}}} + :target {:type "table" + :database (mt/id) + :schema "transforms" + :table "gadget_products"}})] + (mt/user-http-request :rasta :delete 200 (format "ee/transform/%s" (:id resp))) + (prn (mt/user-http-request :rasta :get 500 (format "ee/transform/%s" (:id resp))))))) + +(deftest delete-table-transforms-test + (mt/test-drivers (mt/normal-drivers) + (let [resp (mt/user-http-request :rasta :post 200 "ee/transform" + {:name "Gadget Products" + :source {:type "query" + :query {:database (mt/id) + :type "native", + :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'" + :template-tags {}}}} + :target {:type "table" + :database (mt/id) + :schema "transforms" + :table "gadget_products"}})] + (mt/user-http-request :rasta :delete 200 (format "ee/transform/%s/table" (:id resp)))))) From d87da10b7efae269496e19a312d533f5c4e623c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 19:57:59 +0300 Subject: [PATCH 034/164] [skip ci] Fix table existence check --- .../metabase_enterprise/transforms/api.clj | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 6bc9387aa103..51b6c700b402 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -1,6 +1,5 @@ (ns metabase-enterprise.transforms.api (:require - [clojure.set :as set] [metabase-enterprise.transforms.execute :as transforms.execute] [metabase.api.common :as api] [metabase.api.macros :as api.macros] @@ -42,26 +41,26 @@ :table "gadget_products"}}] -) -;; TODO add driver specific quoting (defn- qualified-table-name - [{:keys [schema table]}] - (cond->> table - schema (str schema "."))) + [driver {:keys [schema table]}] + (cond->> (driver/escape-alias driver table) + (string? schema) (str (driver/escape-alias driver schema) "."))) (defn- target-table-exists? [{:keys [source target] :as _transform}] - (let [database (-> source :query :database) - driver (t2/select-one-fn :engine :model/Database database)] - (-> (driver/describe-table driver database (set/rename-keys target {:table :name})) - :fields - seq - boolean))) + (let [db-id (-> source :query :database) + database (t2/select-one :model/Database db-id) + driver (:engine database) + needle ((juxt :schema :table) target) + normalize-fn (juxt :schema :name)] + (some #(= (normalize-fn %) needle) + (:tables (driver/describe-database driver database))))) (defn- delete-target-table! [{:keys [source target] :as _transform}] (let [database (-> source :query :database) driver (t2/select-one-fn :engine :model/Database database)] - (driver/drop-table! driver database (qualified-table-name target)))) + (driver/drop-table! driver database (qualified-table-name driver target)))) (defn- delete-target-table-by-id! [transform-id] @@ -144,7 +143,7 @@ {:db db :driver driver :sql (compile-source source) - :output-table (qualified-table-name target) + :output-table (qualified-table-name driver target) :overwrite? true}))) (def ^{:arglists '([request respond raise])} routes From c54de85f0a664cbd233873b079cd86d439c4aeb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 20:02:05 +0300 Subject: [PATCH 035/164] [skip ci] Require superuser --- .../src/metabase_enterprise/transforms/api.clj | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 51b6c700b402..51166553e6bf 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -70,6 +70,7 @@ "Get a list of transforms." [_route-params _query-params] + (api/check-superuser) (t2/select :model/Transform)) (api.macros/defendpoint :post "/" @@ -79,6 +80,7 @@ [:name :string] [:source ::transform-source] [:target ::transform-target]]] + (api/check-superuser) (when (target-table-exists? body) (api/throw-403)) (let [id (t2/insert-returning-pk! :model/Transform {:name name @@ -90,6 +92,7 @@ [{:keys [id]} :- [:map [:id ms/PositiveInt]]] (log/info "get transform" id) + (api/check-superuser) (t2/select-one :model/Transform id)) (api.macros/defendpoint :put "/:id" @@ -101,6 +104,7 @@ [:source {:optional true} ::transform-source] [:target {:optional true} ::transform-target]]] (log/info "put transform" id) + (api/check-superuser) (let [old (t2/select-one-fn :target :model/Transform id) new (merge old body)] (when (not= (select-keys (:target old) [:schema :table]) @@ -115,7 +119,8 @@ [{:keys [id]} :- [:map [:id ms/PositiveInt]]] (log/info "delete transform" id) - #_(delete-target-table-by-id! id) + (api/check-superuser) + (delete-target-table-by-id! id) (t2/delete! :model/Transform id) nil) @@ -123,6 +128,7 @@ [{:keys [id]} :- [:map [:id ms/PositiveInt]]] (log/info "delete transform target table" id) + (api/check-superuser) (delete-target-table-by-id! id)) (defn- compile-source [{query-type :type :as source}] @@ -133,12 +139,10 @@ [{:keys [id]} :- [:map [:id ms/PositiveInt]]] (log/info "execute transform" id) - (let [{:keys [_name source target]} (t2/select-one :model/Transform id) + (api/check-superuser) + (let [{:keys [source target]} (t2/select-one :model/Transform id) db (get-in source [:query :database]) {driver :engine} (t2/select-one :model/Database db)] - (when (not= (perms/full-db-permission-for-user api/*current-user-id* :perms/create-queries db) - :query-builder-and-native) - (api/throw-403)) (transforms.execute/execute {:db db :driver driver From bb438cee88c45e66df8dd3bd8e73f06f790b4772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 20:07:16 +0300 Subject: [PATCH 036/164] [skip ci] Remove superfluous require --- enterprise/backend/src/metabase_enterprise/transforms/api.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 51166553e6bf..7b7eca007243 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -6,7 +6,6 @@ [metabase.api.routes.common :refer [+auth]] [metabase.api.util.handlers :as handlers] [metabase.driver :as driver] - [metabase.permissions.core :as perms] [metabase.query-processor.compile :as qp.compile] [metabase.util.log :as log] [metabase.util.malli.registry :as mr] From d49b886527e49dcb56c92cbb47445c3ef95b974d Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Tue, 22 Jul 2025 12:39:04 -0500 Subject: [PATCH 037/164] [ci skip] Use crowberto in tests and fix api --- .../metabase_enterprise/transforms/api.clj | 4 +-- .../transforms/api_test.clj | 32 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 7b7eca007243..91a7253ef521 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -92,7 +92,7 @@ [:id ms/PositiveInt]]] (log/info "get transform" id) (api/check-superuser) - (t2/select-one :model/Transform id)) + (api/check-404 (t2/select-one :model/Transform id))) (api.macros/defendpoint :put "/:id" [{:keys [id]} :- [:map @@ -104,7 +104,7 @@ [:target {:optional true} ::transform-target]]] (log/info "put transform" id) (api/check-superuser) - (let [old (t2/select-one-fn :target :model/Transform id) + (let [old (t2/select-one :model/Transform id) new (merge old body)] (when (not= (select-keys (:target old) [:schema :table]) (select-keys (:target new) [:schema :table])) diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj index 2aabed6f2de4..d78150302384 100644 --- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj +++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj @@ -8,11 +8,11 @@ (deftest list-transforms-test (mt/test-drivers (mt/normal-drivers) - (mt/user-http-request :rasta :get 200 "ee/transform"))) + (mt/user-http-request :crowberto :get 200 "ee/transform"))) (deftest create-transform-test (mt/test-drivers (mt/normal-drivers) - (mt/user-http-request :rasta :post 200 "ee/transform" + (mt/user-http-request :crowberto :post 200 "ee/transform" {:name "Gadget Products" :source {:type "query" :query {:database (mt/id) @@ -27,7 +27,7 @@ (deftest get-transforms-test (mt/test-drivers (mt/normal-drivers) - (let [resp (mt/user-http-request :rasta :post 200 "ee/transform" + (let [resp (mt/user-http-request :crowberto :post 200 "ee/transform" {:name "Gadget Products" :source {:type "query" :query {:database (mt/id) @@ -36,13 +36,13 @@ :template-tags {}}}} :target {:type "table" :database (mt/id) - :schema "transforms" + ;;:schema "transforms" :table "gadget_products"}})] - (mt/user-http-request :rasta :get 200 (format "ee/transform/%s" (:id resp)))))) + (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" (:id resp)))))) (deftest put-transforms-test (mt/test-drivers (mt/normal-drivers) - (let [resp (mt/user-http-request :rasta :post 200 "ee/transform" + (let [resp (mt/user-http-request :crowberto :post 200 "ee/transform" {:name "Gadget Products" :source {:type "query" :query {:database (mt/id) @@ -51,9 +51,9 @@ :template-tags {}}}} :target {:type "table" :database (mt/id) - :schema "transforms" + ;;:schema "transforms" :table "gadget_products"}})] - (mt/user-http-request :rasta :put 200 (format "ee/transform/%s" (:id resp)) + (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp)) {:name "Gadget Products 2" :source {:type "query" :query {:database (mt/id) @@ -62,12 +62,12 @@ :template-tags {}}}} :target {:type "table" :database (mt/id) - :schema "transforms" + ;;:schema "transforms" :table "gadget_products"}})))) (deftest delete-transforms-test (mt/test-drivers (mt/normal-drivers) - (let [resp (mt/user-http-request :rasta :post 200 "ee/transform" + (let [resp (mt/user-http-request :crowberto :post 200 "ee/transform" {:name "Gadget Products" :source {:type "query" :query {:database (mt/id) @@ -76,14 +76,14 @@ :template-tags {}}}} :target {:type "table" :database (mt/id) - :schema "transforms" + ;;:schema "transforms" :table "gadget_products"}})] - (mt/user-http-request :rasta :delete 200 (format "ee/transform/%s" (:id resp))) - (prn (mt/user-http-request :rasta :get 500 (format "ee/transform/%s" (:id resp))))))) + (mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s" (:id resp))) + (mt/user-http-request :crowberto :get 404 (format "ee/transform/%s" (:id resp)))))) (deftest delete-table-transforms-test (mt/test-drivers (mt/normal-drivers) - (let [resp (mt/user-http-request :rasta :post 200 "ee/transform" + (let [resp (mt/user-http-request :crowberto :post 200 "ee/transform" {:name "Gadget Products" :source {:type "query" :query {:database (mt/id) @@ -92,6 +92,6 @@ :template-tags {}}}} :target {:type "table" :database (mt/id) - :schema "transforms" + ;;:schema "transforms" :table "gadget_products"}})] - (mt/user-http-request :rasta :delete 200 (format "ee/transform/%s/table" (:id resp)))))) + (mt/user-http-request :crowberto :delete 200 (format "ee/transform/%s/table" (:id resp)))))) From 821df47bac4037c4276e9b9ea9d1d65929b4b907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 20:51:24 +0300 Subject: [PATCH 038/164] [skip ci] Sync transform output table on execution --- .clj-kondo/config/modules/config.edn | 1 + .../src/metabase_enterprise/transforms/api.clj | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn index 5803803dac4d..19ac9ff6a79a 100644 --- a/.clj-kondo/config/modules/config.edn +++ b/.clj-kondo/config/modules/config.edn @@ -1841,6 +1841,7 @@ models permissions query-processor + sync util}} enterprise/upload-management diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 91a7253ef521..45c92929b148 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -7,6 +7,7 @@ [metabase.api.util.handlers :as handlers] [metabase.driver :as driver] [metabase.query-processor.compile :as qp.compile] + [metabase.sync.core :as sync] [metabase.util.log :as log] [metabase.util.malli.registry :as mr] [metabase.util.malli.schema :as ms] @@ -134,6 +135,16 @@ (case query-type "query" (:query (qp.compile/compile-with-inline-parameters (:query source))))) +(defn- sync-table! + [database target] + (let [table (or (t2/select-one :model/Table + :db_id (:id database) + :schema (:schema target) + :name (:table target)) + (sync/create-table! database {:schema (:schema target) + :name (:table target)}))] + (sync/sync-table! table))) + (api.macros/defendpoint :post "/:id/execute" [{:keys [id]} :- [:map [:id ms/PositiveInt]]] @@ -141,13 +152,14 @@ (api/check-superuser) (let [{:keys [source target]} (t2/select-one :model/Transform id) db (get-in source [:query :database]) - {driver :engine} (t2/select-one :model/Database db)] + {driver :engine :as database} (t2/select-one :model/Database db)] (transforms.execute/execute {:db db :driver driver :sql (compile-source source) :output-table (qualified-table-name driver target) - :overwrite? true}))) + :overwrite? true}) + (sync-table! database target))) (def ^{:arglists '([request respond raise])} routes "`/api/ee/transform` routes." From 9e4bca97ceb115677602f416a9dabbdc70fdd21b Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 22 Jul 2025 14:45:10 -0400 Subject: [PATCH 039/164] Basic page --- .../src/metabase-enterprise/plugins.js | 1 + .../EditorHeader/EditorHeader.tsx | 37 +++++++++++++++++++ .../TransformEditor/EditorHeader/index.ts | 1 + .../TransformEditor/TransformEditor.tsx | 22 +++++++++++ .../components/TransformEditor/index.ts | 1 + .../metabase-enterprise/transforms/index.ts | 5 +++ .../NewQueryTransformPage.tsx | 32 ++++++++++++++++ .../pages/NewQueryTransformPage/index.ts | 1 + .../metabase-enterprise/transforms/types.ts | 9 +++++ frontend/src/metabase/plugins/index.ts | 4 ++ frontend/src/metabase/routes.jsx | 8 ++++ frontend/src/metabase/selectors/app.ts | 1 + 12 files changed, 122 insertions(+) create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/EditorHeader.tsx create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/index.ts create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/index.ts create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/index.ts create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/NewQueryTransformPage.tsx create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/index.ts create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/types.ts diff --git a/enterprise/frontend/src/metabase-enterprise/plugins.js b/enterprise/frontend/src/metabase-enterprise/plugins.js index 02c55793de7d..6e1fcba55d25 100644 --- a/enterprise/frontend/src/metabase-enterprise/plugins.js +++ b/enterprise/frontend/src/metabase-enterprise/plugins.js @@ -39,3 +39,4 @@ import "./user_provisioning"; import "./clean_up"; import "./metabot"; import "./database_replication"; +import "./transforms"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/EditorHeader.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/EditorHeader.tsx new file mode 100644 index 000000000000..7ab969663654 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/EditorHeader.tsx @@ -0,0 +1,37 @@ +import { t } from "ttag"; + +import Button from "metabase/common/components/Button"; +import EditBar from "metabase/common/components/EditBar"; +import type { TransformInfo } from "metabase-enterprise/transforms/types"; + +type EditorHeaderProps = { + transform: TransformInfo; + onCreate: () => void; + onSave: () => void; + onCancel: () => void; +}; + +export function EditorHeader({ + transform, + onCreate, + onSave, + onCancel, +}: EditorHeaderProps) { + return ( + {t`Cancel`}, + transform.id != null ? ( + + ) : ( + + ), + ]} + /> + ); +} diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/index.ts new file mode 100644 index 000000000000..541ab7c1a459 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/index.ts @@ -0,0 +1 @@ +export * from "./EditorHeader"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx new file mode 100644 index 000000000000..548e56d97c19 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx @@ -0,0 +1,22 @@ +import { Flex } from "metabase/ui"; + +import type { TransformInfo } from "../../types"; + +import { EditorHeader } from "./EditorHeader"; + +type TransformEditorProps = { + transform: TransformInfo; +}; + +export function TransformEditor({ transform }: TransformEditorProps) { + return ( + + undefined} + onSave={() => undefined} + onCancel={() => undefined} + /> + + ); +} diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/index.ts new file mode 100644 index 000000000000..7071b94b7607 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/index.ts @@ -0,0 +1 @@ +export * from "./TransformEditor"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/index.ts new file mode 100644 index 000000000000..6e5a513e621b --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/index.ts @@ -0,0 +1,5 @@ +import { PLUGIN_TRANSFORMS } from "metabase/plugins"; + +import { NewQueryTransformPage } from "./pages/NewQueryTransformPage"; + +PLUGIN_TRANSFORMS.NewQueryTransformPage = NewQueryTransformPage; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/NewQueryTransformPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/NewQueryTransformPage.tsx new file mode 100644 index 000000000000..929e0a192d35 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/NewQueryTransformPage.tsx @@ -0,0 +1,32 @@ +import { useState } from "react"; + +import Question from "metabase-lib/v1/Question"; + +import { TransformEditor } from "../../components/TransformEditor"; +import type { TransformInfo } from "../../types"; + +type NewTransformPageParams = { + databaseId?: string; +}; + +type NewQueryTransformPageProps = { + params?: NewTransformPageParams; +}; + +export function NewQueryTransformPage({ + params = {}, +}: NewQueryTransformPageProps) { + const [transform] = useState(() => getInitialTransform(params)); + return ; +} + +function getDatabaseId({ databaseId = "" }: NewTransformPageParams) { + return parseInt(databaseId, 10); +} + +function getInitialTransform(params: NewTransformPageParams): TransformInfo { + const databaseId = getDatabaseId(params); + return { + query: Question.create({ type: "query", databaseId }).datasetQuery(), + }; +} diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/index.ts new file mode 100644 index 000000000000..735ff381b38f --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/index.ts @@ -0,0 +1 @@ +export * from "./NewQueryTransformPage"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/types.ts b/enterprise/frontend/src/metabase-enterprise/transforms/types.ts new file mode 100644 index 000000000000..aacc653e6d35 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/types.ts @@ -0,0 +1,9 @@ +import type { DatasetQuery, TransformId } from "metabase-types/api"; + +export type TransformInfo = { + id?: TransformId; + name?: string; + query: DatasetQuery; + table?: string; + schema?: string; +}; diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.ts index 8d74bb71c336..d2cb302f0a19 100644 --- a/frontend/src/metabase/plugins/index.ts +++ b/frontend/src/metabase/plugins/index.ts @@ -786,3 +786,7 @@ export const PLUGIN_SMTP_OVERRIDE = { CloudSMTPConnectionCard: PluginPlaceholder, SMTPOverrideConnectionForm: PluginPlaceholder, }; + +export const PLUGIN_TRANSFORMS = { + NewQueryTransformPage: PluginPlaceholder, +}; diff --git a/frontend/src/metabase/routes.jsx b/frontend/src/metabase/routes.jsx index 6869c4dc4090..27a7366c32e3 100644 --- a/frontend/src/metabase/routes.jsx +++ b/frontend/src/metabase/routes.jsx @@ -43,6 +43,7 @@ import { PLUGIN_EMBEDDING_IFRAME_SDK_SETUP, PLUGIN_LANDING_PAGE, PLUGIN_METABOT, + PLUGIN_TRANSFORMS, } from "metabase/plugins"; import { QueryBuilder } from "metabase/query_builder/containers/QueryBuilder"; import { loadCurrentUser } from "metabase/redux/user"; @@ -296,6 +297,13 @@ export const getRoutes = (store) => { + + + + diff --git a/frontend/src/metabase/selectors/app.ts b/frontend/src/metabase/selectors/app.ts index 050bd7ef8072..18d646058446 100644 --- a/frontend/src/metabase/selectors/app.ts +++ b/frontend/src/metabase/selectors/app.ts @@ -35,6 +35,7 @@ const PATHS_WITHOUT_NAVBAR = [ /\/metric\/.*\/metadata/, /\/metric\/query/, /\/metric\/metadata/, + /\/transform\/new\/.*\/query/, ]; const PATHS_WITH_COLLECTION_BREADCRUMBS = [ From 1d9c6c71f386e1cb829750a728537091b8628a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Benk=C5=91?= Date: Tue, 22 Jul 2025 21:52:31 +0300 Subject: [PATCH 040/164] [skip ci] Sync transform on create an update too --- .../metabase_enterprise/transforms/api.clj | 77 +++++++++++-------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/api.clj b/enterprise/backend/src/metabase_enterprise/transforms/api.clj index 45c92929b148..1fbea3d95324 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/api.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/api.clj @@ -66,6 +66,33 @@ [transform-id] (delete-target-table! (t2/select-one :model/Transform transform-id))) +(defn- compile-source [{query-type :type :as source}] + (case query-type + "query" (:query (qp.compile/compile-with-inline-parameters (:query source))))) + +(defn- sync-table! + [database target] + (let [table (or (t2/select-one :model/Table + :db_id (:id database) + :schema (:schema target) + :name (:table target)) + (sync/create-table! database {:schema (:schema target) + :name (:table target)}))] + (sync/sync-table! table))) + +(defn- exec-transform + [transform] + (let [{:keys [source target]} transform + db (get-in source [:query :database]) + {driver :engine :as database} (t2/select-one :model/Database db)] + (transforms.execute/execute + {:db db + :driver driver + :sql (compile-source source) + :output-table (qualified-table-name driver target) + :overwrite? true}) + (sync-table! database target))) + (api.macros/defendpoint :get "/" "Get a list of transforms." [_route-params @@ -83,10 +110,11 @@ (api/check-superuser) (when (target-table-exists? body) (api/throw-403)) - (let [id (t2/insert-returning-pk! :model/Transform {:name name - :source source - :target target})] - (t2/select-one :model/Transform id))) + (let [transform (t2/insert-returning-instance! :model/Transform {:name name + :source source + :target target})] + (exec-transform transform) + transform)) (api.macros/defendpoint :get "/:id" [{:keys [id]} :- [:map @@ -106,14 +134,18 @@ (log/info "put transform" id) (api/check-superuser) (let [old (t2/select-one :model/Transform id) - new (merge old body)] - (when (not= (select-keys (:target old) [:schema :table]) - (select-keys (:target new) [:schema :table])) - (when (target-table-exists? new) - (api/throw-403)) + new (merge old body) + target-fields #(-> % :target (select-keys [:schema :table])) + query-fields #(select-keys % [:source :target])] + (when (and (not= (target-fields old) (target-fields new)) + (target-table-exists? new)) + (api/throw-403)) + (when (not= (query-fields new) (query-fields old)) (delete-target-table! old))) (t2/update! :model/Transform id body) - (t2/select-one :model/Transform id)) + (let [transform (t2/select-one :model/Transform id)] + (exec-transform transform) + transform)) (api.macros/defendpoint :delete "/:id" [{:keys [id]} :- [:map @@ -131,35 +163,12 @@ (api/check-superuser) (delete-target-table-by-id! id)) -(defn- compile-source [{query-type :type :as source}] - (case query-type - "query" (:query (qp.compile/compile-with-inline-parameters (:query source))))) - -(defn- sync-table! - [database target] - (let [table (or (t2/select-one :model/Table - :db_id (:id database) - :schema (:schema target) - :name (:table target)) - (sync/create-table! database {:schema (:schema target) - :name (:table target)}))] - (sync/sync-table! table))) - (api.macros/defendpoint :post "/:id/execute" [{:keys [id]} :- [:map [:id ms/PositiveInt]]] (log/info "execute transform" id) (api/check-superuser) - (let [{:keys [source target]} (t2/select-one :model/Transform id) - db (get-in source [:query :database]) - {driver :engine :as database} (t2/select-one :model/Database db)] - (transforms.execute/execute - {:db db - :driver driver - :sql (compile-source source) - :output-table (qualified-table-name driver target) - :overwrite? true}) - (sync-table! database target))) + (exec-transform (t2/select-one :model/Transform id))) (def ^{:arglists '([request respond raise])} routes "`/api/ee/transform` routes." From 1e498de06bdca571079b87337ca76b1376747a93 Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Tue, 22 Jul 2025 14:55:58 -0500 Subject: [PATCH 041/164] [ci skip] lego definition (interpreter) for transform --- .../metabase_enterprise/legos/actions.clj | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 enterprise/backend/test/metabase_enterprise/legos/actions.clj diff --git a/enterprise/backend/test/metabase_enterprise/legos/actions.clj b/enterprise/backend/test/metabase_enterprise/legos/actions.clj new file mode 100644 index 000000000000..cc375ccab42e --- /dev/null +++ b/enterprise/backend/test/metabase_enterprise/legos/actions.clj @@ -0,0 +1,42 @@ +(ns metabase-enterprise.legos.actions + (:require + [metabase-enterprise.transforms.execute :as transforms.execute] + [metabase.driver :as driver] + [metabase.util.malli.registry :as mr] + [toucan2.core :as t2])) + +(defn- dispatch-execute [& [lego]] + (keyword (:lego lego))) + +(defmulti execute! + "Execute a lego description." + {:added "0.55.0" :arglists '([lego])} + #'dispatch-execute) + +(defmethod execute! :default + [lego] + (throw (ex-info (str "legos.actions/execute! is not implemented for " (:lego lego)) + {:lego lego}))) + +(mr/def ::transform + [:map + [:lego [:= "transform"]] + [:database :int] + [:schema {:optional true} :string] + [:table :string] + [:query :string]]) + +(defn- qualified-table-name + [driver schema table] + (cond->> (driver/escape-alias driver table) + (string? schema) (str (driver/escape-alias driver schema) "."))) + +(defmethod execute! :transform + [{:keys [database schema table query]}] + (let [{driver :engine :as database} (t2/select-one :model/Database database)] + (transforms.execute/execute + {:db database + :driver driver + :sql query + :output-table (qualified-table-name driver schema table) + :overwrite? true}))) From d2f02f7846c8fb4a67868a22d3f526840c8ac213 Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Tue, 22 Jul 2025 15:03:00 -0500 Subject: [PATCH 042/164] [ci skip] Add plan --- .../test/metabase_enterprise/legos/actions.clj | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/enterprise/backend/test/metabase_enterprise/legos/actions.clj b/enterprise/backend/test/metabase_enterprise/legos/actions.clj index cc375ccab42e..6c1139f9e2fa 100644 --- a/enterprise/backend/test/metabase_enterprise/legos/actions.clj +++ b/enterprise/backend/test/metabase_enterprise/legos/actions.clj @@ -18,6 +18,10 @@ (throw (ex-info (str "legos.actions/execute! is not implemented for " (:lego lego)) {:lego lego}))) +(mr/def ::lego + [:map + [:lego :string]]) + (mr/def ::transform [:map [:lego [:= "transform"]] @@ -40,3 +44,13 @@ :sql query :output-table (qualified-table-name driver schema table) :overwrite? true}))) + +(mr/def ::plan + [:map + [:steps [:+ ::lego]]]) + +(defn execute-plan! + "Execute an entire plan." + [plan] + (doseq [step (:steps plan)] + (execute! step))) From 7443a2d8bc025d7616f750fde5c6dec375aa92b3 Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Tue, 22 Jul 2025 15:11:17 -0500 Subject: [PATCH 043/164] [ci skip] Add provisional transfer code --- .../test/metabase_enterprise/legos/actions.clj | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/enterprise/backend/test/metabase_enterprise/legos/actions.clj b/enterprise/backend/test/metabase_enterprise/legos/actions.clj index 6c1139f9e2fa..720da60b86d9 100644 --- a/enterprise/backend/test/metabase_enterprise/legos/actions.clj +++ b/enterprise/backend/test/metabase_enterprise/legos/actions.clj @@ -37,7 +37,7 @@ (defmethod execute! :transform [{:keys [database schema table query]}] - (let [{driver :engine :as database} (t2/select-one :model/Database database)] + (let [{driver :engine} (t2/select-one :model/Database database)] (transforms.execute/execute {:db database :driver driver @@ -45,6 +45,21 @@ :output-table (qualified-table-name driver schema table) :overwrite? true}))) +(mr/def ::transfer + [:map + [:lego [:= "transfer"]] + [:source_database :int] + [:destination_database :int] + [:source_table :string] + [:destination_table :string]]) + +(defmethod execute! :transfer + [{:keys [source_database destination_database + source_table destination_table]}] + (let [{driver :engine :as database} (t2/select-one :model/Database source_database)] + ;; do the right thing 👍 🕶️ + )) + (mr/def ::plan [:map [:steps [:+ ::lego]]]) From 2585f352a13bc55677693e663e42bece6d78e1ce Mon Sep 17 00:00:00 2001 From: Eric Normand Date: Tue, 22 Jul 2025 16:04:21 -0500 Subject: [PATCH 044/164] [ci skip] test edn, json, yaml versions --- .../legos/actions_test.clj | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 enterprise/backend/test/metabase_enterprise/legos/actions_test.clj diff --git a/enterprise/backend/test/metabase_enterprise/legos/actions_test.clj b/enterprise/backend/test/metabase_enterprise/legos/actions_test.clj new file mode 100644 index 000000000000..d7fdcd4035c0 --- /dev/null +++ b/enterprise/backend/test/metabase_enterprise/legos/actions_test.clj @@ -0,0 +1,55 @@ +(ns metabase-enterprise.legos.actions-test + (:require + [clojure.test :refer :all] + [metabase-enterprise.legos.actions :as legos.actions] + [metabase.test :as mt] + [metabase.util.json :as json] + [metabase.util.yaml :as yaml])) + +(deftest execute-transform-test + (mt/test-drivers (mt/normal-drivers) + (testing "execute transform edn" + (is (legos.actions/execute! {:lego "transform" + :database (mt/id) + :table "TARGET_TABLE" + :query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'"}))) + + (testing "execute transform json" + (is (legos.actions/execute! + (json/decode (format " +{ + \"lego\": \"transform\", + \"database\": %d, + \"table\": \"TARGET_TABLE\", + \"query\": \"SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'\" +} +" (mt/id)) + keyword)))) + + (testing "execute transform yaml" + (is (legos.actions/execute! + (yaml/parse-string + (format " +lego: transform +database: %d +table: TARGET_TABLE +query: SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget' +" (mt/id)))))))) + +(deftest execute-plan-test + (mt/test-drivers (mt/normal-drivers) + (testing "executing a plan!" + (is (legos.actions/execute-plan! + (yaml/parse-string + (format " +steps: + - lego: transform + database: %d + table: TABLEA + query: SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget' + - lego: transform + database: %d + table: TABLEB + query: SELECT *, average(RATING) as AR FROM TABLE_A GROUP BY DATE_TRUNC('month', CREATED_AT) + +" (mt/id) (mt/id)))))))) From f7d947ff72ff8388288aedb75d5476cd0b4065e2 Mon Sep 17 00:00:00 2001 From: William Zimrin Date: Tue, 22 Jul 2025 18:18:03 -0500 Subject: [PATCH 045/164] basic transfers work [ci skip] --- .clj-kondo/config/modules/config.edn | 9 ++ .../src/metabase_enterprise/core/init.clj | 1 + .../metabase_enterprise/transfers/core.clj | 1 + .../metabase_enterprise/transfers/execute.clj | 90 +++++++++++++++++++ .../metabase_enterprise/transfers/init.clj | 1 + .../transforms/execute.clj | 13 +-- src/metabase/driver.clj | 2 +- src/metabase/driver/sql/query_processor.clj | 14 ++- 8 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 enterprise/backend/src/metabase_enterprise/transfers/core.clj create mode 100644 enterprise/backend/src/metabase_enterprise/transfers/execute.clj create mode 100644 enterprise/backend/src/metabase_enterprise/transfers/init.clj diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn index 19ac9ff6a79a..416d39b80c7c 100644 --- a/.clj-kondo/config/modules/config.edn +++ b/.clj-kondo/config/modules/config.edn @@ -1844,6 +1844,15 @@ sync util}} + enterprise/transfers + {:team "Querying" + :api #{metabase-enterprise.transforms.api + metabase-enterprise.transforms.init} + :uses #{driver + query-processor + util + transforms}} + enterprise/upload-management {:team "Admin Webapp" :api #{metabase-enterprise.upload-management.api} diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj index 005a18de0fdb..fe79cc5b0e1f 100644 --- a/enterprise/backend/src/metabase_enterprise/core/init.clj +++ b/enterprise/backend/src/metabase_enterprise/core/init.clj @@ -16,4 +16,5 @@ [metabase-enterprise.scim.init] [metabase-enterprise.sso.init] [metabase-enterprise.stale.init] + [metabase-enterprise.tranfers.init] [metabase-enterprise.transforms.init])) diff --git a/enterprise/backend/src/metabase_enterprise/transfers/core.clj b/enterprise/backend/src/metabase_enterprise/transfers/core.clj new file mode 100644 index 000000000000..b69d78a86beb --- /dev/null +++ b/enterprise/backend/src/metabase_enterprise/transfers/core.clj @@ -0,0 +1 @@ +(ns metabase-enterprise.transfers.core) diff --git a/enterprise/backend/src/metabase_enterprise/transfers/execute.clj b/enterprise/backend/src/metabase_enterprise/transfers/execute.clj new file mode 100644 index 000000000000..3790162ab1ed --- /dev/null +++ b/enterprise/backend/src/metabase_enterprise/transfers/execute.clj @@ -0,0 +1,90 @@ +(ns metabase-enterprise.transfers.execute + (:require [honey.sql.helpers :as sql.helpers] + [metabase-enterprise.transforms.execute :as transforms.execute] + [metabase.driver.postgres :as pg] + [metabase.driver.sql.query-processor :as sql.qp] + [metabase.query-processor :as qp] + [metabase.query-processor.compile :as qp.compile] + [toucan2.core :as t2])) + +(def ^:private type-map + "Map of Field base types -> Postgres column types. + + Created by manually reversing the postgres driver default-base-types map and resolving + conflicts by picking the most 'general' postgres type." + {:type/BigInteger :bigint + :type/Boolean :boolean + :type/Decimal :decimal + :type/Float (keyword "double precision") + :type/Integer :integer + :type/IPAddress :inet + :type/JSON :jsonb + :type/Structured :text + :type/Text :text + :type/Date :date + :type/DateTime :timestamp + :type/DateTimeWithLocalTZ :timestamptz + :type/Time :time + :type/TimeWithLocalTZ :timetz + :type/UUID :uuid}) + +(defn- make-table [{:keys [input-fields output-db-ref output-table-name overwrite?]}] + (let [fields (for [{:keys [base_type nfc_path parent_id] :as field} input-fields + :let [pg-type (type-map base_type)] + :when (and (not nfc_path) (not parent_id) pg-type)] + (assoc field :pg-type pg-type))] + (when overwrite? + (transforms.execute/execute-query :postgres output-db-ref (driver/compile-drop-table :postgres (keyword output-table-name)))) + + (-> (sql.helpers/create-table (keyword output-table-name)) + (sql.helpers/with-columns (for [{:keys [name pg-type]} fields] + [[:raw name] pg-type])) + (->> (sql.qp/format-honeysql :postgres)) + (->> (transforms.execute/execute-query :postgres output-db-ref))) + fields)) + +(def #^:private step-size 100) + +(defn- get-data [input-table fields] + (-> {:database (:db_id input-table) + :type :query + :query {:source-table (:id input-table) + :fields (mapv (fn [{:keys [id]}] + [:field id]) + fields)} + :middleware {:disable-remaps? true}} + qp/process-query + :data + :rows + (->> (partition step-size)))) + +(defn- cast-row [row fields] + (map (fn [val {:keys [pg-type name]}] + [:cast val pg-type]) + row + fields)) + +(defn- insert-data [{:keys [data output-db-ref fields output-table-name]}] + (let [cast-data (map (fn [row] + (cast-row row fields)) + data) + query (-> (sql.helpers/insert-into (keyword output-table-name)) + (as-> query (apply sql.helpers/columns query (map (fn [{:keys [name]}] + [:raw name]) + fields))) + (sql.helpers/values cast-data) + (->> (sql.qp/format-honeysql :postgres)))] + (transforms.execute/execute-query :postgres output-db-ref query))) + +(defn execute [{:keys [input-table output-db-ref output-table-name overwrite?] :as args}] + (let [input-fields (t2/select :model/Field :table_id (:id input-table)) + fields (make-table {:input-fields input-fields + :output-db-ref output-db-ref + :output-table-name output-table-name + :overwrite? overwrite?}) + data (get-data input-table fields)] + (doseq [page data] + (insert-data {:data page + :output-db-ref output-db-ref + :output-table-name output-table-name + :fields fields})))) diff --git a/enterprise/backend/src/metabase_enterprise/transfers/init.clj b/enterprise/backend/src/metabase_enterprise/transfers/init.clj new file mode 100644 index 000000000000..d5abba27eb9e --- /dev/null +++ b/enterprise/backend/src/metabase_enterprise/transfers/init.clj @@ -0,0 +1 @@ +(ns metabase-enterprise.transfers.init) diff --git a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj index 4a4a7e316501..818e6b54b5a6 100644 --- a/enterprise/backend/src/metabase_enterprise/transforms/execute.clj +++ b/enterprise/backend/src/metabase_enterprise/transforms/execute.clj @@ -5,18 +5,19 @@ [metabase.query-processor.preprocess :as qp.preprocess] [metabase.query-processor.setup :as qp.setup])) -(defn- execute-query [driver db sql] - (let [query {:native {:query sql} +(defn execute-query [driver db-ref [sql & params]] + (let [query {:native (cond-> {:query sql} + params (assoc :params params)) :type :native - :database db}] + :database db-ref}] (qp.setup/with-qp-setup [query query] (let [query (qp.preprocess/preprocess query)] (driver/execute-write-query! driver query))))) -(defn execute [{:keys [db driver sql output-table overwrite?]}] +(defn execute [{:keys [db-ref driver sql output-table overwrite?]}] (let [output-table (keyword (or output-table (str "transform_" (str/replace (random-uuid) \- \_)))) query (driver/compile-transform driver {:sql sql :output-table output-table :overwrite? overwrite?})] (when overwrite? - (execute-query driver db (driver/drop-transform driver output-table))) - (execute-query driver db query) + (execute-query driver db-ref (driver/compile-drop-table driver output-table))) + (execute-query driver db-ref query) output-table)) diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj index 78b3afbd939f..a7bcb0ed8a6a 100644 --- a/src/metabase/driver.clj +++ b/src/metabase/driver.clj @@ -1356,4 +1356,4 @@ (defmulti compile-transform dispatch-on-initialized-driver :hierarchy #'hierarchy) -(defmulti drop-transform dispatch-on-initialized-driver :hierarchy #'hierarchy) +(defmulti compile-drop-table dispatch-on-initialized-driver :hierarchy #'hierarchy) diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj index 4b04ec44346c..5ed7d5f32c2c 100644 --- a/src/metabase/driver/sql/query_processor.clj +++ b/src/metabase/driver/sql/query_processor.clj @@ -2075,13 +2075,11 @@ (defmethod driver/compile-transform :sql [driver {:keys [sql output-table]}] - (first - (format-honeysql driver - {:select [:*] - :from [[[:raw (str "(" sql ")")] (str (random-uuid))]] - :into output-table}))) + (format-honeysql driver + {:select [:*] + :from [[[:raw (str "(" sql ")")] (str (random-uuid))]] + :into output-table})) -(defmethod driver/drop-transform :sql +(defmethod driver/compile-drop-table :sql [driver table] - (first - (format-honeysql driver {:drop-table [:if-exists table]}))) + (format-honeysql driver {:drop-table [:if-exists table]})) From 461519495c7a8627056519b7dd845414744cf894 Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 22 Jul 2025 21:23:46 -0400 Subject: [PATCH 046/164] Basic page --- .../src/metabase-enterprise/plugins.js | 1 - .../EditorHeader/EditorHeader.tsx | 37 -------------- .../TransformEditor/EditorHeader/index.ts | 1 - .../TransformEditor/TransformEditor.tsx | 22 -------- .../components/TransformEditor/index.ts | 1 - .../metabase-enterprise/transforms/index.ts | 5 -- .../NewQueryTransformPage.tsx | 32 ------------ .../pages/NewQueryTransformPage/index.ts | 1 - .../metabase-enterprise/transforms/types.ts | 9 ---- .../components/TablePicker/Results.tsx | 3 +- .../DataModel/components/TablePicker/types.ts | 21 ++++++-- .../DataModel/components/TablePicker/utils.ts | 50 +++++++++++++------ 12 files changed, 55 insertions(+), 128 deletions(-) delete mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/EditorHeader.tsx delete mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/index.ts delete mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx delete mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/index.ts delete mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/index.ts delete mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/NewQueryTransformPage.tsx delete mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/index.ts delete mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/types.ts diff --git a/enterprise/frontend/src/metabase-enterprise/plugins.js b/enterprise/frontend/src/metabase-enterprise/plugins.js index 6e1fcba55d25..02c55793de7d 100644 --- a/enterprise/frontend/src/metabase-enterprise/plugins.js +++ b/enterprise/frontend/src/metabase-enterprise/plugins.js @@ -39,4 +39,3 @@ import "./user_provisioning"; import "./clean_up"; import "./metabot"; import "./database_replication"; -import "./transforms"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/EditorHeader.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/EditorHeader.tsx deleted file mode 100644 index 7ab969663654..000000000000 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/EditorHeader.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { t } from "ttag"; - -import Button from "metabase/common/components/Button"; -import EditBar from "metabase/common/components/EditBar"; -import type { TransformInfo } from "metabase-enterprise/transforms/types"; - -type EditorHeaderProps = { - transform: TransformInfo; - onCreate: () => void; - onSave: () => void; - onCancel: () => void; -}; - -export function EditorHeader({ - transform, - onCreate, - onSave, - onCancel, -}: EditorHeaderProps) { - return ( - {t`Cancel`}, - transform.id != null ? ( - - ) : ( - - ), - ]} - /> - ); -} diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/index.ts deleted file mode 100644 index 541ab7c1a459..000000000000 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/EditorHeader/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./EditorHeader"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx deleted file mode 100644 index 548e56d97c19..000000000000 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import { Flex } from "metabase/ui"; - -import type { TransformInfo } from "../../types"; - -import { EditorHeader } from "./EditorHeader"; - -type TransformEditorProps = { - transform: TransformInfo; -}; - -export function TransformEditor({ transform }: TransformEditorProps) { - return ( - - undefined} - onSave={() => undefined} - onCancel={() => undefined} - /> - - ); -} diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/index.ts deleted file mode 100644 index 7071b94b7607..000000000000 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./TransformEditor"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/index.ts deleted file mode 100644 index 6e5a513e621b..000000000000 --- a/enterprise/frontend/src/metabase-enterprise/transforms/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { PLUGIN_TRANSFORMS } from "metabase/plugins"; - -import { NewQueryTransformPage } from "./pages/NewQueryTransformPage"; - -PLUGIN_TRANSFORMS.NewQueryTransformPage = NewQueryTransformPage; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/NewQueryTransformPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/NewQueryTransformPage.tsx deleted file mode 100644 index 929e0a192d35..000000000000 --- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/NewQueryTransformPage.tsx +++ /dev/null @@ -1,32 +0,0 @@ -import { useState } from "react"; - -import Question from "metabase-lib/v1/Question"; - -import { TransformEditor } from "../../components/TransformEditor"; -import type { TransformInfo } from "../../types"; - -type NewTransformPageParams = { - databaseId?: string; -}; - -type NewQueryTransformPageProps = { - params?: NewTransformPageParams; -}; - -export function NewQueryTransformPage({ - params = {}, -}: NewQueryTransformPageProps) { - const [transform] = useState(() => getInitialTransform(params)); - return ; -} - -function getDatabaseId({ databaseId = "" }: NewTransformPageParams) { - return parseInt(databaseId, 10); -} - -function getInitialTransform(params: NewTransformPageParams): TransformInfo { - const databaseId = getDatabaseId(params); - return { - query: Question.create({ type: "query", databaseId }).datasetQuery(), - }; -} diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/index.ts deleted file mode 100644 index 735ff381b38f..000000000000 --- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewQueryTransformPage/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./NewQueryTransformPage"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/types.ts b/enterprise/frontend/src/metabase-enterprise/transforms/types.ts deleted file mode 100644 index aacc653e6d35..000000000000 --- a/enterprise/frontend/src/metabase-enterprise/transforms/types.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { DatasetQuery, TransformId } from "metabase-types/api"; - -export type TransformInfo = { - id?: TransformId; - name?: string; - query: DatasetQuery; - table?: string; - schema?: string; -}; diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.tsx b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.tsx index f24192ffc94c..5850ec3fa0b6 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.tsx +++ b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.tsx @@ -190,7 +190,8 @@ export function Results({ justify="space-between" gap="sm" ref={virtual.measureElement} - className={cx(S.item, S[type], { + className={cx(S.item, { + [S.table]: type === "table", [S.active]: isActive, [S.selected]: selectedIndex === index, })} diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/types.ts b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/types.ts index 061def2b8e08..3ed7b36797fa 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/types.ts +++ b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/types.ts @@ -11,9 +11,15 @@ export type TreePath = { databaseId?: DatabaseId; schemaName?: SchemaName; tableId?: TableId; + sectionId?: "transform"; }; -export type TreeNode = RootNode | DatabaseNode | SchemaNode | TableNode; +export type TreeNode = + | RootNode + | DatabaseNode + | SchemaNode + | TableNode + | TransformListNode; export type RootNode = { type: "root"; @@ -28,7 +34,7 @@ export type DatabaseNode = { label: string; key: NodeKey; value: { databaseId: DatabaseId }; - children: SchemaNode[]; + children: (SchemaNode | TransformListNode)[]; }; export type SchemaNode = { @@ -49,11 +55,20 @@ export type TableNode = { disabled?: boolean; }; +export type TransformListNode = { + type: "transform-list"; + key: NodeKey; + label: string; + value: { databaseId: DatabaseId; sectionId: "transform" }; + children: []; +}; + export type DatabaseItem = Omit; export type SchemaItem = Omit; export type TableItem = Omit; +export type TransformListItem = Omit; -export type Item = DatabaseItem | SchemaItem | TableItem; +export type Item = DatabaseItem | SchemaItem | TableItem | TransformListItem; export type ItemType = Item["type"]; diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts index 934ebbc5c8c0..dd881c2344e3 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts +++ b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts @@ -1,5 +1,6 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { useDeepCompareEffect, useLatest } from "react-use"; +import { t } from "ttag"; import _ from "underscore"; import { @@ -24,6 +25,7 @@ import type { RootNode, SchemaNode, TableNode, + TransformListNode, TreeNode, TreePath, } from "./types"; @@ -33,12 +35,14 @@ const CHILD_TYPES = { database: "schema", schema: "table", table: null, + "transform-list": null, } as const; export const TYPE_ICONS: Record = { table: "table2", schema: "folder", database: "database", + "transform-list": "add_data", }; export function hasChildren(type: ItemType): boolean { @@ -198,13 +202,20 @@ export function useTableLoader(path: TreePath) { children: database.value.databaseId !== databaseId ? database.children - : schemas.map((schema) => ({ - ...schema, - children: - schema.value.schemaName !== schemaName - ? schema.children - : tables, - })), + : [ + node({ + type: "transform-list", + label: t`Transforms`, + value: { databaseId, sectionId: "transform" }, + }), + ...schemas.map((schema) => ({ + ...schema, + children: + schema.value.schemaName !== schemaName + ? schema.children + : tables, + })), + ], })), ); setTree((current) => { @@ -270,9 +281,10 @@ export function useSearch(query: string) { tree.children.push(databaseNode); } - let schemaNode = databaseNode.children.find((node) => { - return node.type === "schema" && node.value.schemaName === tableSchema; - }); + let schemaNode = databaseNode.children.find( + (node): node is SchemaNode => + node.type === "schema" && node.value.schemaName === tableSchema, + ); if (!schemaNode) { schemaNode = node({ type: "schema", @@ -317,13 +329,15 @@ export function useSearch(query: string) { export function useExpandedState(path: TreePath) { const [state, setState] = useState(expandPath({}, path)); - const { databaseId, schemaName, tableId } = path; + const { databaseId, schemaName, tableId, sectionId } = path; useEffect(() => { // When the path changes, this means a user has navigated throught the browser back // button, ensure the path is completely expanded. - setState((state) => expandPath(state, { databaseId, schemaName, tableId })); - }, [databaseId, schemaName, tableId]); + setState((state) => + expandPath(state, { databaseId, schemaName, tableId, sectionId }), + ); + }, [databaseId, schemaName, tableId, sectionId]); const isExpanded = useCallback( (path: string | TreePath) => { @@ -359,6 +373,12 @@ function expandPath(state: ExpandedState, path: TreePath): ExpandedState { tableId: undefined, schemaName: undefined, })]: true, + [toKey({ + ...path, + tableId: undefined, + schemaName: undefined, + sectionId: undefined, + })]: true, [toKey({ ...path, tableId: undefined, @@ -492,8 +512,8 @@ function merge(a: TreeNode | undefined, b: TreeNode | undefined): TreeNode { /** * Create a unique key for a TreePath */ -function toKey({ databaseId, schemaName, tableId }: TreePath) { - return JSON.stringify([databaseId, schemaName, tableId]); +function toKey({ databaseId, schemaName, tableId, sectionId }: TreePath) { + return JSON.stringify([databaseId, schemaName, tableId, sectionId]); } type Optional = Omit & Partial>; From b340f8ed7d72791166e89d41cda1b15f1a4a187a Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 22 Jul 2025 22:33:38 -0400 Subject: [PATCH 047/164] Basic page --- .../src/metabase-enterprise/api/index.ts | 1 + .../src/metabase-enterprise/api/tags.ts | 17 ++ .../src/metabase-enterprise/api/transform.ts | 74 ++++++ .../src/metabase-enterprise/plugins.js | 1 + .../transforms/hooks/use-fetch-transforms.ts | 7 + .../metabase-enterprise/transforms/index.ts | 5 + frontend/src/metabase/admin/routes.jsx | 1 + .../browse/transforms/BrowseTransforms.tsx | 59 ----- .../browse/transforms/TransformsTable.tsx | 213 ------------------ .../src/metabase/browse/transforms/index.ts | 1 - .../src/metabase/browse/transforms/types.ts | 1 - .../components/FieldSection/utils.tsx | 1 + .../DataModel/components/TablePicker/types.ts | 25 +- .../DataModel/components/TablePicker/utils.ts | 71 ++++-- .../metadata/pages/DataModel/types.ts | 11 +- .../metadata/pages/DataModel/utils.ts | 13 +- frontend/src/metabase/plugins/index.ts | 22 +- .../components/QueryModals/QueryModals.tsx | 7 +- frontend/src/metabase/routes.jsx | 2 - .../NewTransformModal/NewTransformModal.tsx | 109 --------- .../components/NewTransformModal/index.ts | 1 - 21 files changed, 231 insertions(+), 411 deletions(-) create mode 100644 enterprise/frontend/src/metabase-enterprise/api/transform.ts create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/hooks/use-fetch-transforms.ts create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/index.ts delete mode 100644 frontend/src/metabase/browse/transforms/BrowseTransforms.tsx delete mode 100644 frontend/src/metabase/browse/transforms/TransformsTable.tsx delete mode 100644 frontend/src/metabase/browse/transforms/index.ts delete mode 100644 frontend/src/metabase/browse/transforms/types.ts delete mode 100644 frontend/src/metabase/transforms/components/NewTransformModal/NewTransformModal.tsx delete mode 100644 frontend/src/metabase/transforms/components/NewTransformModal/index.ts diff --git a/enterprise/frontend/src/metabase-enterprise/api/index.ts b/enterprise/frontend/src/metabase-enterprise/api/index.ts index deb2c93ccf2a..a589d39302eb 100644 --- a/enterprise/frontend/src/metabase-enterprise/api/index.ts +++ b/enterprise/frontend/src/metabase-enterprise/api/index.ts @@ -10,4 +10,5 @@ export * from "./db-routing"; export * from "./saml"; export * from "./scim"; export * from "./tags"; +export * from "./transform"; export * from "./upload-management"; diff --git a/enterprise/frontend/src/metabase-enterprise/api/tags.ts b/enterprise/frontend/src/metabase-enterprise/api/tags.ts index 524efcda423f..25d0dd86b522 100644 --- a/enterprise/frontend/src/metabase-enterprise/api/tags.ts +++ b/enterprise/frontend/src/metabase-enterprise/api/tags.ts @@ -1,11 +1,16 @@ import type { TagDescription } from "@reduxjs/toolkit/query"; +import { TAG_TYPES } from "metabase/api/tags"; +import type { Transform } from "metabase-types/api"; + export const ENTERPRISE_TAG_TYPES = [ + ...TAG_TYPES, "scim", "metabot", "metabot-entities-list", "metabot-prompt-suggestions", "gsheets-status", + "transform", ] as const; export type EnterpriseTagType = (typeof ENTERPRISE_TAG_TYPES)[number]; @@ -35,3 +40,15 @@ export function invalidateTags( ): TagDescription[] { return !error ? tags : []; } + +export function provideTransformTags( + transform: Transform, +): TagDescription[] { + return [idTag("transform", transform.id)]; +} + +export function provideTransformListTags( + transforms: Transform[], +): TagDescription[] { + return [listTag("transform"), ...transforms.flatMap(provideTransformTags)]; +} diff --git a/enterprise/frontend/src/metabase-enterprise/api/transform.ts b/enterprise/frontend/src/metabase-enterprise/api/transform.ts new file mode 100644 index 000000000000..321dda2c936f --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/api/transform.ts @@ -0,0 +1,74 @@ +import type { + CreateTransformRequest, + Transform, + TransformId, +} from "metabase-types/api"; + +import { EnterpriseApi } from "./api"; +import { + idTag, + invalidateTags, + listTag, + provideTransformListTags, + provideTransformTags, + tag, +} from "./tags"; + +export const transformApi = EnterpriseApi.injectEndpoints({ + endpoints: (builder) => ({ + listTransforms: builder.query({ + query: (params) => ({ + method: "GET", + url: "/api/ee/transform", + params, + }), + providesTags: (transforms = []) => provideTransformListTags(transforms), + }), + getTransform: builder.query({ + query: (id) => ({ + method: "GET", + url: `/api/ee/transform/${id}`, + }), + providesTags: (transform) => + transform ? provideTransformTags(transform) : [], + }), + createTransform: builder.mutation({ + query: (body) => ({ + method: "POST", + url: "/api/ee/transform", + body, + }), + invalidatesTags: (_, error) => + invalidateTags(error, [listTag("transform"), tag("transform")]), + }), + executeTransform: builder.mutation({ + query: (id) => ({ + method: "POST", + url: `/api/ee/transform/${id}/execute`, + }), + invalidatesTags: (_, error) => + invalidateTags(error, [ + tag("table"), + tag("field"), + tag("field-values"), + ]), + }), + deleteTransform: builder.mutation({ + query: (id) => ({ + method: "DELETE", + url: `/api/ee/transform/${id}`, + }), + invalidatesTags: (_, error, id) => + invalidateTags(error, [listTag("transform"), idTag("transform", id)]), + }), + }), +}); + +export const { + useListTransformsQuery, + useLazyListTransformsQuery, + useGetTransformQuery, + useCreateTransformMutation, + useExecuteTransformMutation, + useDeleteTransformMutation, +} = transformApi; diff --git a/enterprise/frontend/src/metabase-enterprise/plugins.js b/enterprise/frontend/src/metabase-enterprise/plugins.js index 02c55793de7d..6e1fcba55d25 100644 --- a/enterprise/frontend/src/metabase-enterprise/plugins.js +++ b/enterprise/frontend/src/metabase-enterprise/plugins.js @@ -39,3 +39,4 @@ import "./user_provisioning"; import "./clean_up"; import "./metabot"; import "./database_replication"; +import "./transforms"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/hooks/use-fetch-transforms.ts b/enterprise/frontend/src/metabase-enterprise/transforms/hooks/use-fetch-transforms.ts new file mode 100644 index 000000000000..7e64a0fea478 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/hooks/use-fetch-transforms.ts @@ -0,0 +1,7 @@ +import type { UseFetchTransformsResult } from "metabase/plugins"; +import { useLazyListTransformsQuery } from "metabase-enterprise/api"; + +export function useFetchTransforms(): UseFetchTransformsResult { + const [fetchTransforms] = useLazyListTransformsQuery(); + return [fetchTransforms]; +} diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/index.ts new file mode 100644 index 000000000000..f8b95db4d504 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/index.ts @@ -0,0 +1,5 @@ +import { PLUGIN_TRANSFORMS } from "metabase/plugins"; + +import { useFetchTransforms } from "./hooks/use-fetch-transforms"; + +PLUGIN_TRANSFORMS.useFetchTransforms = useFetchTransforms; diff --git a/frontend/src/metabase/admin/routes.jsx b/frontend/src/metabase/admin/routes.jsx index c7401c6ae1fa..3556ddb4baac 100644 --- a/frontend/src/metabase/admin/routes.jsx +++ b/frontend/src/metabase/admin/routes.jsx @@ -75,6 +75,7 @@ const getRoutes = (store, CanAccessSettings, IsAdmin) => ( + - - - - - <Group gap="sm"> - <Icon - size={24} - color="var(--mb-color-icon-primary)" - name="function" - /> - {t`Transforms`} - </Group> - - - - - - - - } - > - - - - - - - ); -} diff --git a/frontend/src/metabase/browse/transforms/TransformsTable.tsx b/frontend/src/metabase/browse/transforms/TransformsTable.tsx deleted file mode 100644 index 7abae3f8b53c..000000000000 --- a/frontend/src/metabase/browse/transforms/TransformsTable.tsx +++ /dev/null @@ -1,213 +0,0 @@ -import { type MouseEvent, useMemo } from "react"; -import { t } from "ttag"; - -import { - useDeleteTransformMutation, - useExecuteTransformMutation, -} from "metabase/api"; -import EntityItem from "metabase/common/components/EntityItem"; -import { SortableColumnHeader } from "metabase/common/components/ItemsTable/BaseItemsTable"; -import { - ColumnHeader, - ItemNameCell, - MaybeItemLink, - TBody, - Table, - TableColumn, -} from "metabase/common/components/ItemsTable/BaseItemsTable.styled"; -import { Columns } from "metabase/common/components/ItemsTable/Columns"; -import { - Button, - Icon, - type IconName, - Menu, - Repeat, - Skeleton, -} from "metabase/ui"; -import type { Transform } from "metabase-types/api"; - -import { Cell, NameColumn, TableRow } from "../components/BrowseTable.styled"; - -type TransformsTableProps = { - transforms?: Transform[]; - skeleton?: boolean; -}; - -export const itemsTableContainerName = "ItemsTableContainer"; - -const sharedProps = { - containerName: itemsTableContainerName, -}; - -const nameProps = { - ...sharedProps, -}; - -const menuProps = { - ...sharedProps, -}; - -const DOTMENU_WIDTH = 34; - -export function TransformsTable({ - transforms = [], - skeleton = false, -}: TransformsTableProps) { - return ( - - - - - - - - - - {t`Name`} - - - - - - - {skeleton ? ( - - - - ) : ( - transforms.map((transform: Transform) => ( - - )) - )} - -
- ); -} - -function TransformRow({ transform }: { transform?: Transform }) { - return ( - - - - - - ); -} - -function SkeletonText() { - return ; -} - -function stopPropagation(event: MouseEvent) { - event.stopPropagation(); -} - -function preventDefault(event: MouseEvent) { - event.preventDefault(); -} - -function NameCell({ transform }: { transform?: Transform }) { - const headingId = `transform-${transform?.id ?? "dummy"}-heading`; - - return ( - - - paddingInlineStart: "1.4rem", - paddingInlineEnd: ".5rem", - }} - onClick={preventDefault} - > - {transform ? ( - - ) : ( - - )} - - - ); -} - -type TransformAction = { - key: string; - title: string; - icon: IconName; - action: () => void; -}; - -function MenuCell({ transform }: { transform?: Transform }) { - const [executeTransformMutation] = useExecuteTransformMutation(); - const [deleteTransformMutation] = useDeleteTransformMutation(); - - const actions = useMemo(() => { - if (!transform) { - return []; - } - - const actions: TransformAction[] = []; - actions.push({ - key: "execute", - title: t`Execute`, - icon: "play", - action: () => executeTransformMutation(transform.id), - }); - actions.push({ - key: "delete", - title: t`Delete`, - icon: "trash", - action: () => deleteTransformMutation(transform.id), - }); - - return actions; - }, [transform, executeTransformMutation, deleteTransformMutation]); - - return ( - - - - - - - {actions.map((action) => ( - } - onClick={action.action} - > - {action.title} - - ))} - - - - ); -} diff --git a/frontend/src/metabase/browse/transforms/index.ts b/frontend/src/metabase/browse/transforms/index.ts deleted file mode 100644 index 3adcfe7e9e60..000000000000 --- a/frontend/src/metabase/browse/transforms/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./BrowseTransforms"; diff --git a/frontend/src/metabase/browse/transforms/types.ts b/frontend/src/metabase/browse/transforms/types.ts deleted file mode 100644 index 777e5a5dde09..000000000000 --- a/frontend/src/metabase/browse/transforms/types.ts +++ /dev/null @@ -1 +0,0 @@ -export type SortColumn = "name"; diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/FieldSection/utils.tsx b/frontend/src/metabase/metadata/pages/DataModel/components/FieldSection/utils.tsx index 19b6e8bc69fa..34a2623ea713 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/FieldSection/utils.tsx +++ b/frontend/src/metabase/metadata/pages/DataModel/components/FieldSection/utils.tsx @@ -39,6 +39,7 @@ export function getSemanticTypeError( const parentField = fieldsByName[parentName]; const href = getUrl({ databaseId: table.db_id, + sectionId: undefined, schemaName: table.schema, tableId: table.id, fieldId: getRawTableFieldId(field), diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/types.ts b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/types.ts index 3ed7b36797fa..c7f50ae6087f 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/types.ts +++ b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/types.ts @@ -3,6 +3,7 @@ import type { SchemaName, Table, TableId, + TransformId, } from "metabase-types/api"; export type NodeKey = string; @@ -12,6 +13,7 @@ export type TreePath = { schemaName?: SchemaName; tableId?: TableId; sectionId?: "transform"; + transformId?: TransformId; }; export type TreeNode = @@ -19,6 +21,7 @@ export type TreeNode = | DatabaseNode | SchemaNode | TableNode + | TransformNode | TransformListNode; export type RootNode = { @@ -55,20 +58,38 @@ export type TableNode = { disabled?: boolean; }; +export type TransformNode = { + type: "transform"; + key: NodeKey; + label: string; + value: { + databaseId: DatabaseId; + sectionId: "transform"; + transformId: TransformId; + }; + children: []; +}; + export type TransformListNode = { type: "transform-list"; key: NodeKey; label: string; value: { databaseId: DatabaseId; sectionId: "transform" }; - children: []; + children: TransformNode[]; }; export type DatabaseItem = Omit; export type SchemaItem = Omit; export type TableItem = Omit; +export type TransformItem = Omit; export type TransformListItem = Omit; -export type Item = DatabaseItem | SchemaItem | TableItem | TransformListItem; +export type Item = + | DatabaseItem + | SchemaItem + | TableItem + | TransformItem + | TransformListItem; export type ItemType = Item["type"]; diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts index dd881c2344e3..c7e596bb6c0d 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts +++ b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts @@ -11,9 +11,11 @@ import { useSearchQuery, } from "metabase/api"; import { isSyncCompleted } from "metabase/lib/syncing"; +import { PLUGIN_TRANSFORMS } from "metabase/plugins"; import type { IconName } from "metabase/ui"; import type { DatabaseId, SchemaName } from "metabase-types/api"; +import type { SectionId } from "../../types"; import { getUrl as getUrl_ } from "../../utils"; import type { @@ -26,6 +28,7 @@ import type { SchemaNode, TableNode, TransformListNode, + TransformNode, TreeNode, TreePath, } from "./types"; @@ -35,26 +38,29 @@ const CHILD_TYPES = { database: "schema", schema: "table", table: null, - "transform-list": null, + transform: null, + "transform-list": "transform", } as const; export const TYPE_ICONS: Record = { table: "table2", schema: "folder", database: "database", + transform: "refresh_downstream", "transform-list": "add_data", }; export function hasChildren(type: ItemType): boolean { - return type !== "table"; + return type !== "table" && type !== "transform"; } export function getUrl(value: TreePath) { return getUrl_({ - fieldId: undefined, - tableId: undefined, databaseId: undefined, + sectionId: undefined, schemaName: undefined, + tableId: undefined, + fieldId: undefined, ...value, }); } @@ -74,6 +80,7 @@ export function useTableLoader(path: TreePath) { const [fetchDatabases, databases] = useLazyListDatabasesQuery(); const [fetchSchemas, schemas] = useLazyListDatabaseSchemasQuery(); const [fetchTables, tables] = useLazyListDatabaseSchemaTablesQuery(); + const [fetchTransforms] = PLUGIN_TRANSFORMS.useFetchTransforms(); const databasesRef = useLatest(databases); const schemasRef = useLatest(schemas); const tablesRef = useLatest(tables); @@ -103,6 +110,31 @@ export function useTableLoader(path: TreePath) { ); }, [fetchDatabases, databasesRef]); + const getTransforms = useCallback( + async ( + databaseId: DatabaseId | undefined, + sectionId: SectionId | undefined, + ) => { + if (databaseId === undefined || sectionId !== "transform") { + return []; + } + + const response = await fetchTransforms(); + return response?.data?.map((transform) => + node({ + type: "transform", + label: transform.name, + value: { + databaseId, + sectionId: "transform", + transformId: transform.id, + }, + }), + ); + }, + [fetchTransforms], + ); + const getTables = useCallback( async ( databaseId: DatabaseId | undefined, @@ -190,10 +222,11 @@ export function useTableLoader(path: TreePath) { const load = useCallback( async function (path: TreePath) { const { databaseId, schemaName } = path; - const [databases, schemas, tables] = await Promise.all([ + const [databases, schemas, tables, transforms] = await Promise.all([ getDatabases(), getSchemas(path.databaseId), getTables(path.databaseId, path.schemaName), + getTransforms(path.databaseId, path.sectionId), ]); const newTree: TreeNode = rootNode( @@ -207,6 +240,7 @@ export function useTableLoader(path: TreePath) { type: "transform-list", label: t`Transforms`, value: { databaseId, sectionId: "transform" }, + children: transforms, }), ...schemas.map((schema) => ({ ...schema, @@ -223,7 +257,7 @@ export function useTableLoader(path: TreePath) { return _.isEqual(current, merged) ? current : merged; }); }, - [getDatabases, getSchemas, getTables], + [getDatabases, getSchemas, getTables, getTransforms], ); useDeepCompareEffect(() => { @@ -373,12 +407,6 @@ function expandPath(state: ExpandedState, path: TreePath): ExpandedState { tableId: undefined, schemaName: undefined, })]: true, - [toKey({ - ...path, - tableId: undefined, - schemaName: undefined, - sectionId: undefined, - })]: true, [toKey({ ...path, tableId: undefined, @@ -473,6 +501,9 @@ export function flatten( function sort(nodes: TreeNode[]): TreeNode[] { return Array.from(nodes).sort((a, b) => { + if (a.type === "transform-list" || b.type === "transform-list") { + return a.type === "transform-list" ? -1 : 1; + } return a.label.localeCompare(b.label); }); } @@ -512,8 +543,20 @@ function merge(a: TreeNode | undefined, b: TreeNode | undefined): TreeNode { /** * Create a unique key for a TreePath */ -function toKey({ databaseId, schemaName, tableId, sectionId }: TreePath) { - return JSON.stringify([databaseId, schemaName, tableId, sectionId]); +function toKey({ + databaseId, + schemaName, + tableId, + sectionId, + transformId, +}: TreePath) { + return JSON.stringify([ + databaseId, + schemaName, + tableId, + sectionId, + transformId, + ]); } type Optional = Omit & Partial>; diff --git a/frontend/src/metabase/metadata/pages/DataModel/types.ts b/frontend/src/metabase/metadata/pages/DataModel/types.ts index 2f0a5879b081..d11d38f3524c 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/types.ts +++ b/frontend/src/metabase/metadata/pages/DataModel/types.ts @@ -4,22 +4,29 @@ import type { SchemaId, SchemaName, TableId, + TransformId, } from "metabase-types/api"; export type RouteParams = { databaseId?: string; - fieldId?: string; schemaId?: SchemaId; tableId?: string; + fieldId?: string; + sectionId?: string; + transformId?: string; }; export type ParsedRouteParams = { databaseId: DatabaseId | undefined; - fieldId: FieldId | undefined; schemaName: SchemaName | undefined; tableId: TableId | undefined; + fieldId: FieldId | undefined; + sectionId: SectionId | undefined; + transformId: TransformId | undefined; }; +export type SectionId = "transform"; + export type Column = "nav" | "table" | "field" | "preview"; export interface ColumnSizeConfig { diff --git a/frontend/src/metabase/metadata/pages/DataModel/utils.ts b/frontend/src/metabase/metadata/pages/DataModel/utils.ts index 09249a47e1ff..a246d7c1548b 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/utils.ts +++ b/frontend/src/metabase/metadata/pages/DataModel/utils.ts @@ -15,11 +15,14 @@ export function parseRouteParams(params: RouteParams): ParsedRouteParams { : params.schemaId, tableId: Urls.extractEntityId(params.tableId), fieldId: Urls.extractEntityId(params.fieldId), + sectionId: params.sectionId === "transform" ? "transform" : undefined, + transformId: Urls.extractEntityId(params.transformId), }; } export function getUrl(params: ParsedRouteParams): string { - const { databaseId, schemaName, tableId, fieldId } = params; + const { databaseId, schemaName, tableId, fieldId, sectionId, transformId } = + params; const schemaId = `${databaseId}:${schemaName}`; if ( @@ -39,6 +42,14 @@ export function getUrl(params: ParsedRouteParams): string { return `/admin/datamodel/database/${databaseId}/schema/${schemaId}`; } + if (databaseId != null && sectionId != null && transformId != null) { + return `/admin/datamodel/database/${databaseId}/${sectionId}/${transformId}`; + } + + if (databaseId != null && sectionId != null) { + return `/admin/datamodel/database/${databaseId}/${sectionId}`; + } + if (databaseId != null) { return `/admin/datamodel/database/${databaseId}`; } diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.ts index d2cb302f0a19..dff6bd14ae67 100644 --- a/frontend/src/metabase/plugins/index.ts +++ b/frontend/src/metabase/plugins/index.ts @@ -79,6 +79,7 @@ import type { TableId, Timeline, TimelineEvent, + Transform, User, VisualizationDisplay, } from "metabase-types/api"; @@ -787,6 +788,23 @@ export const PLUGIN_SMTP_OVERRIDE = { SMTPOverrideConnectionForm: PluginPlaceholder, }; -export const PLUGIN_TRANSFORMS = { - NewQueryTransformPage: PluginPlaceholder, +export type UseFetchTransformsData = { + data?: Transform[]; +}; + +export type UseFetchTransformsResult = [() => Promise]; + +export type NewTransformFromQueryModalProps = { + question: Question; + onClose: () => void; +}; + +export type TransformsPlugin = { + useFetchTransforms(): UseFetchTransformsResult; + NewTransformFromQueryModal: ComponentType; +}; + +export const PLUGIN_TRANSFORMS: TransformsPlugin = { + useFetchTransforms: () => [() => Promise.resolve({})], + NewTransformFromQueryModal: PluginPlaceholder, }; diff --git a/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx b/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx index 59268495c70e..b56f1ced1836 100644 --- a/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx +++ b/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx @@ -12,6 +12,7 @@ import EntityCopyModal from "metabase/entities/containers/EntityCopyModal"; import { useDispatch, useSelector } from "metabase/lib/redux"; import * as Urls from "metabase/lib/urls"; import { CreateOrEditQuestionAlertModal } from "metabase/notifications/modals"; +import { PLUGIN_TRANSFORMS } from "metabase/plugins"; import { ImpossibleToCreateModelModal } from "metabase/query_builder/components/ImpossibleToCreateModelModal"; import { NewDatasetModal } from "metabase/query_builder/components/NewDatasetModal"; import { QuestionEmbedWidget } from "metabase/query_builder/components/QuestionEmbedWidget"; @@ -23,7 +24,6 @@ import ArchiveQuestionModal from "metabase/questions/containers/ArchiveQuestionM import EditEventModal from "metabase/timelines/questions/containers/EditEventModal"; import MoveEventModal from "metabase/timelines/questions/containers/MoveEventModal"; import NewEventModal from "metabase/timelines/questions/containers/NewEventModal"; -import { NewTransformModal } from "metabase/transforms/components/NewTransformModal"; import type Question from "metabase-lib/v1/Question"; import type { Card, DashboardTabId } from "metabase-types/api"; import type { QueryBuilderMode } from "metabase-types/store"; @@ -340,9 +340,8 @@ export function QueryModals({ ); case MODAL_TYPES.NEW_TRANSFORM: return ( - ); diff --git a/frontend/src/metabase/routes.jsx b/frontend/src/metabase/routes.jsx index 27a7366c32e3..fda596b3524b 100644 --- a/frontend/src/metabase/routes.jsx +++ b/frontend/src/metabase/routes.jsx @@ -16,7 +16,6 @@ import { BrowseSchemas, BrowseTables, } from "metabase/browse"; -import { BrowseTransforms } from "metabase/browse/transforms"; import { ArchiveCollectionModal } from "metabase/collections/components/ArchiveCollectionModal"; import CollectionLanding from "metabase/collections/components/CollectionLanding"; import { MoveCollectionModal } from "metabase/collections/components/MoveCollectionModal"; @@ -308,7 +307,6 @@ export const getRoutes = (store) => { - void; -}; - -export function NewTransformModal({ - query, - opened, - onClose, -}: NewTransformModalProps) { - return ( - - - - - {t`New transform`} - - - - - - - - - - ); -} - -type NewTransformFormProps = { - query: Lib.Query; - onClose: () => void; -}; - -type NewTransformSettings = { - name: string; - schema: string; - table: string; -}; - -const NEW_TRANSFORM_SCHEMA = Yup.object().shape({ - name: Yup.string().required(Errors.required).default(""), - schema: Yup.string().required(Errors.required).default(""), - table: Yup.string().required(Errors.required).default(""), -}); - -function NewTransformForm({ query, onClose }: NewTransformFormProps) { - const [createTransform] = useCreateTransformMutation(); - - const handleSubmit = async (settings: NewTransformSettings) => { - await createTransform(getRequest(query, settings)).unwrap(); - onClose(); - }; - - return ( - -
- - - - - - - - - -
-
- ); -} - -function getRequest( - query: Lib.Query, - settings: NewTransformSettings, -): CreateTransformRequest { - return { - name: settings.name, - source: { - type: "query", - query: Lib.toLegacyQuery(query), - }, - target: { - type: "table", - database: Lib.databaseIdOrThrow(query), - schema: settings.schema, - table: settings.table, - }, - }; -} diff --git a/frontend/src/metabase/transforms/components/NewTransformModal/index.ts b/frontend/src/metabase/transforms/components/NewTransformModal/index.ts deleted file mode 100644 index 60cee20bd3aa..000000000000 --- a/frontend/src/metabase/transforms/components/NewTransformModal/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./NewTransformModal"; From f941af8a1446ac91eba76d26ec332c2f18b50290 Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 22 Jul 2025 22:53:14 -0400 Subject: [PATCH 048/164] Basic page --- .../transforms/hooks/use-fetch-transforms.ts | 4 ++-- .../DataModel/components/TablePicker/utils.ts | 14 +++++++++++--- frontend/src/metabase/plugins/index.ts | 11 +++++++++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/hooks/use-fetch-transforms.ts b/enterprise/frontend/src/metabase-enterprise/transforms/hooks/use-fetch-transforms.ts index 7e64a0fea478..39be83958e73 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/hooks/use-fetch-transforms.ts +++ b/enterprise/frontend/src/metabase-enterprise/transforms/hooks/use-fetch-transforms.ts @@ -2,6 +2,6 @@ import type { UseFetchTransformsResult } from "metabase/plugins"; import { useLazyListTransformsQuery } from "metabase-enterprise/api"; export function useFetchTransforms(): UseFetchTransformsResult { - const [fetchTransforms] = useLazyListTransformsQuery(); - return [fetchTransforms]; + const [fetchTransforms, fetchState] = useLazyListTransformsQuery(); + return [fetchTransforms, fetchState]; } diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts index c7e596bb6c0d..484ff47bffc6 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts +++ b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/utils.ts @@ -57,10 +57,11 @@ export function hasChildren(type: ItemType): boolean { export function getUrl(value: TreePath) { return getUrl_({ databaseId: undefined, - sectionId: undefined, schemaName: undefined, tableId: undefined, fieldId: undefined, + sectionId: undefined, + transformId: undefined, ...value, }); } @@ -80,10 +81,11 @@ export function useTableLoader(path: TreePath) { const [fetchDatabases, databases] = useLazyListDatabasesQuery(); const [fetchSchemas, schemas] = useLazyListDatabaseSchemasQuery(); const [fetchTables, tables] = useLazyListDatabaseSchemaTablesQuery(); - const [fetchTransforms] = PLUGIN_TRANSFORMS.useFetchTransforms(); + const [fetchTransforms, transforms] = PLUGIN_TRANSFORMS.useFetchTransforms(); const databasesRef = useLatest(databases); const schemasRef = useLatest(schemas); const tablesRef = useLatest(tables); + const transformsRef = useLatest(transforms); const [tree, setTree] = useState(rootNode()); @@ -119,6 +121,12 @@ export function useTableLoader(path: TreePath) { return []; } + if (transformsRef.current.isError) { + // Do not refetch when this call failed previously. + // This is to prevent infinite data-loading loop as RTK query does not cache error responses. + return []; + } + const response = await fetchTransforms(); return response?.data?.map((transform) => node({ @@ -132,7 +140,7 @@ export function useTableLoader(path: TreePath) { }), ); }, - [fetchTransforms], + [fetchTransforms, transformsRef], ); const getTables = useCallback( diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.ts index dff6bd14ae67..27efb15eff8d 100644 --- a/frontend/src/metabase/plugins/index.ts +++ b/frontend/src/metabase/plugins/index.ts @@ -792,7 +792,14 @@ export type UseFetchTransformsData = { data?: Transform[]; }; -export type UseFetchTransformsResult = [() => Promise]; +export type UseFetchTransformsState = { + isError: boolean; +}; + +export type UseFetchTransformsResult = [ + () => Promise, + UseFetchTransformsState, +]; export type NewTransformFromQueryModalProps = { question: Question; @@ -805,6 +812,6 @@ export type TransformsPlugin = { }; export const PLUGIN_TRANSFORMS: TransformsPlugin = { - useFetchTransforms: () => [() => Promise.resolve({})], + useFetchTransforms: () => [() => Promise.resolve({}), { isError: false }], NewTransformFromQueryModal: PluginPlaceholder, }; From 397e369a2c947b5491cedbb320bd6475218e6726 Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 22 Jul 2025 23:46:11 -0400 Subject: [PATCH 049/164] Basic page --- .../TransformSection/TransformSection.tsx | 40 ++++++++++ .../components/TransformSection/index.ts | 1 + .../metabase-enterprise/transforms/index.ts | 2 + frontend/src/metabase/admin/routes.jsx | 6 +- frontend/src/metabase/api/index.ts | 1 - frontend/src/metabase/api/tags/constants.ts | 1 - frontend/src/metabase/api/tags/utils.ts | 13 ---- frontend/src/metabase/api/transform.ts | 73 ------------------- .../metadata/pages/DataModel/DataModel.tsx | 12 ++- .../components/TablePicker/Results.module.css | 2 +- .../components/TablePicker/Results.tsx | 13 +++- .../components/TablePicker/wrappers.tsx | 5 +- frontend/src/metabase/plugins/index.ts | 13 +++- .../components/QueryModals/QueryModals.tsx | 2 +- 14 files changed, 85 insertions(+), 99 deletions(-) create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx create mode 100644 enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/index.ts delete mode 100644 frontend/src/metabase/api/transform.ts diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx new file mode 100644 index 000000000000..2c2f6a9e7c4b --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx @@ -0,0 +1,40 @@ +import { t } from "ttag"; + +import { NameDescriptionInput } from "metabase/metadata/components/NameDescriptionInput"; +import type { TransformSectionProps } from "metabase/plugins"; +import { Stack } from "metabase/ui"; +import { useGetTransformQuery } from "metabase-enterprise/api"; +import type { Transform } from "metabase-types/api"; + +export function TransformSection({ transformId }: TransformSectionProps) { + const { data: transform } = useGetTransformQuery(transformId); + + if (!transform) { + return null; + } + + return ; +} + +type TransformSettingsProps = { + transform: Transform; +}; + +function TransformSettings({ transform }: TransformSettingsProps) { + return ( + + + undefined} + onDescriptionChange={() => undefined} + /> + + + ); +} diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/index.ts new file mode 100644 index 000000000000..98c9c45d1431 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/index.ts @@ -0,0 +1 @@ +export * from "./TransformSection"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/index.ts index f8b95db4d504..a6624da954ae 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/index.ts +++ b/enterprise/frontend/src/metabase-enterprise/transforms/index.ts @@ -1,5 +1,7 @@ import { PLUGIN_TRANSFORMS } from "metabase/plugins"; +import { TransformSection } from "./components/TransformSection"; import { useFetchTransforms } from "./hooks/use-fetch-transforms"; PLUGIN_TRANSFORMS.useFetchTransforms = useFetchTransforms; +PLUGIN_TRANSFORMS.TransformSection = TransformSection; diff --git a/frontend/src/metabase/admin/routes.jsx b/frontend/src/metabase/admin/routes.jsx index 3556ddb4baac..896d9eb1ee3d 100644 --- a/frontend/src/metabase/admin/routes.jsx +++ b/frontend/src/metabase/admin/routes.jsx @@ -75,7 +75,6 @@ const getRoutes = (store, CanAccessSettings, IsAdmin) => ( - ( path="database/:databaseId/schema/:schemaId/table/:tableId/field/:fieldId" component={DataModel} /> + + diff --git a/frontend/src/metabase/api/index.ts b/frontend/src/metabase/api/index.ts index 217457a97009..0b6a9fc0c055 100644 --- a/frontend/src/metabase/api/index.ts +++ b/frontend/src/metabase/api/index.ts @@ -35,6 +35,5 @@ export * from "./table"; export * from "./task"; export * from "./timeline"; export * from "./timeline-event"; -export * from "./transform"; export * from "./user-key-value"; export * from "./user"; diff --git a/frontend/src/metabase/api/tags/constants.ts b/frontend/src/metabase/api/tags/constants.ts index 135f6e18cdaa..87a6ee00dad7 100644 --- a/frontend/src/metabase/api/tags/constants.ts +++ b/frontend/src/metabase/api/tags/constants.ts @@ -34,7 +34,6 @@ export const TAG_TYPES = [ "task", "timeline", "timeline-event", - "transform", "user", "public-dashboard", "embed-dashboard", diff --git a/frontend/src/metabase/api/tags/utils.ts b/frontend/src/metabase/api/tags/utils.ts index ca052717727f..8612c53920a9 100644 --- a/frontend/src/metabase/api/tags/utils.ts +++ b/frontend/src/metabase/api/tags/utils.ts @@ -40,7 +40,6 @@ import type { Task, Timeline, TimelineEvent, - Transform, UserInfo, WritebackAction, } from "metabase-types/api"; @@ -606,18 +605,6 @@ export function provideTaskListTags(tasks: Task[]): TagDescription[] { return [listTag("task"), ...tasks.flatMap(provideTaskTags)]; } -export function provideTransformTags( - transform: Transform, -): TagDescription[] { - return [idTag("transform", transform.id)]; -} - -export function provideTransformListTags( - transforms: Transform[], -): TagDescription[] { - return [listTag("transform"), ...transforms.flatMap(provideTransformTags)]; -} - export function provideUniqueTasksListTags(): TagDescription[] { return [listTag("unique-tasks")]; } diff --git a/frontend/src/metabase/api/transform.ts b/frontend/src/metabase/api/transform.ts deleted file mode 100644 index 55b4558911d0..000000000000 --- a/frontend/src/metabase/api/transform.ts +++ /dev/null @@ -1,73 +0,0 @@ -import type { - CreateTransformRequest, - Transform, - TransformId, -} from "metabase-types/api"; - -import { Api } from "./api"; -import { - idTag, - invalidateTags, - listTag, - provideTransformListTags, - provideTransformTags, - tag, -} from "./tags"; - -export const transformApi = Api.injectEndpoints({ - endpoints: (builder) => ({ - listTransforms: builder.query({ - query: (params) => ({ - method: "GET", - url: "/api/ee/transform", - params, - }), - providesTags: (transforms = []) => provideTransformListTags(transforms), - }), - getTransform: builder.query({ - query: (id) => ({ - method: "GET", - url: `/api/ee/transform/${id}`, - }), - providesTags: (transform) => - transform ? provideTransformTags(transform) : [], - }), - createTransform: builder.mutation({ - query: (body) => ({ - method: "POST", - url: "/api/ee/transform", - body, - }), - invalidatesTags: (_, error) => - invalidateTags(error, [listTag("transform"), tag("transform")]), - }), - executeTransform: builder.mutation({ - query: (id) => ({ - method: "POST", - url: `/api/ee/transform/${id}/execute`, - }), - invalidatesTags: (_, error) => - invalidateTags(error, [ - tag("table"), - tag("field"), - tag("field-values"), - ]), - }), - deleteTransform: builder.mutation({ - query: (id) => ({ - method: "DELETE", - url: `/api/ee/transform/${id}`, - }), - invalidatesTags: (_, error, id) => - invalidateTags(error, [listTag("transform"), idTag("transform", id)]), - }), - }), -}); - -export const { - useListTransformsQuery, - useGetTransformQuery, - useCreateTransformMutation, - useExecuteTransformMutation, - useDeleteTransformMutation, -} = transformApi; diff --git a/frontend/src/metabase/metadata/pages/DataModel/DataModel.tsx b/frontend/src/metabase/metadata/pages/DataModel/DataModel.tsx index 338f534a3543..a7e5a3a60639 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/DataModel.tsx +++ b/frontend/src/metabase/metadata/pages/DataModel/DataModel.tsx @@ -12,6 +12,7 @@ import { import EmptyState from "metabase/common/components/EmptyState"; import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper"; import { getRawTableFieldId } from "metabase/metadata/utils/field"; +import { PLUGIN_TRANSFORMS } from "metabase/plugins"; import { Box, Flex, Stack, rem } from "metabase/ui"; import S from "./DataModel.module.css"; @@ -37,13 +38,16 @@ interface Props { } export const DataModel = ({ children, location, params }: Props) => { - const { databaseId, fieldId, schemaName, tableId } = parseRouteParams(params); + const { databaseId, fieldId, schemaName, tableId, transformId } = + parseRouteParams(params); const { data: databasesData, isLoading: isLoadingDatabases } = useListDatabasesQuery({ include_editable_data_model: true }); const databaseExists = databasesData?.data?.some( (database) => database.id === databaseId, ); const isSegments = location.pathname.startsWith("/admin/datamodel/segment"); + const isTransforms = transformId != null; + const isTables = !isSegments && !isTransforms; const [isPreviewOpen, { close: closePreview, toggle: togglePreview }] = useDisclosure(); const [isSyncModalOpen, { close: closeSyncModal, open: openSyncModal }] = @@ -117,7 +121,11 @@ export const DataModel = ({ children, location, params }: Props) => { {isSegments && children} - {!isSegments && ( + {isTransforms && ( + + )} + + {isTables && ( <> {databaseId != null && tableId == null && diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.module.css b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.module.css index b7414da0b572..7e5c1dd9ee95 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.module.css +++ b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.module.css @@ -31,7 +31,7 @@ } } - &.table { + &.selectable { padding-left: var(--mantine-spacing-md); } } diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.tsx b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.tsx index 5850ec3fa0b6..9a3b58c88ecd 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.tsx +++ b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/Results.tsx @@ -36,6 +36,7 @@ export function Results({ onSelectedIndexChange, }: Props) { const [activeTableId, setActiveTableId] = useState(path.tableId); + const [activeTransformId, setActiveTransformId] = useState(path.transformId); const ref = useRef(null); const virtual = useVirtualizer({ @@ -102,7 +103,10 @@ export function Results({ parent, disabled, } = item; - const isActive = type === "table" && value?.tableId === activeTableId; + const isActive = + (type === "table" && value?.tableId === activeTableId) || + (type === "transform" && value?.transformId === activeTransformId); + const canBeSelected = type === "table" || type === "transform"; const parentIndex = items.findIndex((item) => item.key === parent); const children = items.filter((item) => item.parent === key); const hasTableChildren = children.some( @@ -122,6 +126,11 @@ export function Results({ if (type === "table") { setActiveTableId(value?.tableId); + setActiveTransformId(undefined); + } + if (type === "transform") { + setActiveTableId(undefined); + setActiveTransformId(value?.transformId); } }; @@ -191,7 +200,7 @@ export function Results({ gap="sm" ref={virtual.measureElement} className={cx(S.item, { - [S.table]: type === "table", + [S.selectable]: canBeSelected, [S.active]: isActive, [S.selected]: selectedIndex === index, })} diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/wrappers.tsx b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/wrappers.tsx index 3854bc77665f..096ed2fb6a02 100644 --- a/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/wrappers.tsx +++ b/frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/wrappers.tsx @@ -20,7 +20,10 @@ export function RouterTablePicker(props: TreePath) { // Update URL only when either opening a table or no table has been opened yet. // We want to keep user looking at a table when navigating databases/schemas. - const canUpdateUrl = value.tableId != null || props.tableId == null; + const canUpdateUrl = + value.tableId != null || + value.transformId != null || + (props.tableId == null && props.transformId == null); if (canUpdateUrl) { if (options?.isAutomatic) { diff --git a/frontend/src/metabase/plugins/index.ts b/frontend/src/metabase/plugins/index.ts index 27efb15eff8d..ea2fc660f352 100644 --- a/frontend/src/metabase/plugins/index.ts +++ b/frontend/src/metabase/plugins/index.ts @@ -80,6 +80,7 @@ import type { Timeline, TimelineEvent, Transform, + TransformId, User, VisualizationDisplay, } from "metabase-types/api"; @@ -801,17 +802,23 @@ export type UseFetchTransformsResult = [ UseFetchTransformsState, ]; -export type NewTransformFromQueryModalProps = { +export type NewTransformModalProps = { question: Question; onClose: () => void; }; +export type TransformSectionProps = { + transformId: TransformId; +}; + export type TransformsPlugin = { useFetchTransforms(): UseFetchTransformsResult; - NewTransformFromQueryModal: ComponentType; + TransformSection: ComponentType; + NewTransformModal: ComponentType; }; export const PLUGIN_TRANSFORMS: TransformsPlugin = { useFetchTransforms: () => [() => Promise.resolve({}), { isError: false }], - NewTransformFromQueryModal: PluginPlaceholder, + TransformSection: PluginPlaceholder, + NewTransformModal: PluginPlaceholder, }; diff --git a/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx b/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx index b56f1ced1836..165d7026bb23 100644 --- a/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx +++ b/frontend/src/metabase/query_builder/components/QueryModals/QueryModals.tsx @@ -340,7 +340,7 @@ export function QueryModals({ ); case MODAL_TYPES.NEW_TRANSFORM: return ( - From a254fc146279ceabb131b0c29d806945bbb47b00 Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Wed, 23 Jul 2025 00:01:31 -0400 Subject: [PATCH 050/164] Basic page --- .../TransformSection/TransformSection.tsx | 62 ++++++++++++++----- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx index 2c2f6a9e7c4b..065414cfb45f 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx @@ -2,7 +2,15 @@ import { t } from "ttag"; import { NameDescriptionInput } from "metabase/metadata/components/NameDescriptionInput"; import type { TransformSectionProps } from "metabase/plugins"; -import { Stack } from "metabase/ui"; +import { + Button, + Card, + Group, + Stack, + Text, + TextInputBlurChange, + Title, +} from "metabase/ui"; import { useGetTransformQuery } from "metabase-enterprise/api"; import type { Transform } from "metabase-types/api"; @@ -22,19 +30,45 @@ type TransformSettingsProps = { function TransformSettings({ transform }: TransformSettingsProps) { return ( - - - undefined} - onDescriptionChange={() => undefined} - /> - + + undefined} + onDescriptionChange={() => undefined} + /> + + + + {t`Generated table settings`} + {t`Each transform creates a table in this database.`} + + undefined} + /> + undefined} + /> + + + + + + ); } From a753f2ed0a953546ff9e4098e12c471028c16af4 Mon Sep 17 00:00:00 2001 From: William Zimrin Date: Wed, 23 Jul 2025 02:23:27 -0500 Subject: [PATCH 051/164] use original case --- .../backend/src/metabase_enterprise/transfers/execute.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enterprise/backend/src/metabase_enterprise/transfers/execute.clj b/enterprise/backend/src/metabase_enterprise/transfers/execute.clj index 3790162ab1ed..f18a6aab206a 100644 --- a/enterprise/backend/src/metabase_enterprise/transfers/execute.clj +++ b/enterprise/backend/src/metabase_enterprise/transfers/execute.clj @@ -38,7 +38,7 @@ (-> (sql.helpers/create-table (keyword output-table-name)) (sql.helpers/with-columns (for [{:keys [name pg-type]} fields] - [[:raw name] pg-type])) + [(keyword name) pg-type])) (->> (sql.qp/format-honeysql :postgres)) (->> (transforms.execute/execute-query :postgres output-db-ref))) fields)) @@ -70,7 +70,7 @@ data) query (-> (sql.helpers/insert-into (keyword output-table-name)) (as-> query (apply sql.helpers/columns query (map (fn [{:keys [name]}] - [:raw name]) + (keyword name)) fields))) (sql.helpers/values cast-data) (->> (sql.qp/format-honeysql :postgres)))] From 259ca021b80a7848b015bad26432ada064f30e7b Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Wed, 23 Jul 2025 08:47:56 -0400 Subject: [PATCH 052/164] Basic page --- .../TransformSection/TransformSection.tsx | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx index 065414cfb45f..dfdc31794ee7 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx @@ -30,45 +30,51 @@ type TransformSettingsProps = { function TransformSettings({ transform }: TransformSettingsProps) { return ( - - undefined} - onDescriptionChange={() => undefined} - /> - - - - {t`Generated table settings`} - {t`Each transform creates a table in this database.`} + + + undefined} + onDescriptionChange={() => undefined} + /> + + + + + {t`Generated table settings`} + {t`Each transform creates a table in this database.`} + + + + undefined} + /> + undefined} + /> - undefined} - /> - undefined} - /> - - - - - - + + + + + {t`Schedule`} + + + + + + + + ); } From cbaa8830753afaa325889f8e0d5c50feebd6fa46 Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Wed, 23 Jul 2025 08:49:09 -0400 Subject: [PATCH 053/164] Basic page --- .../components/TransformSection/TransformSection.tsx | 2 ++ frontend/src/metabase/routes.jsx | 8 -------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx index dfdc31794ee7..94cc17bc05e3 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx @@ -6,6 +6,7 @@ import { Button, Card, Group, + Select, Stack, Text, TextInputBlurChange, @@ -69,6 +70,7 @@ function TransformSettings({ transform }: TransformSettingsProps) { {t`Schedule`} + + {t`Schedule`} + diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/constants.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/constants.ts index 7e134f1f36cb..8244a6fbbe1b 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/constants.ts +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/constants.ts @@ -1,12 +1,6 @@ import { t } from "ttag"; export const SCHEDULE_OPTIONS = [ - { - value: "", - get label() { - return t`Never`; - }, - }, { value: "0 0 0/1 * * ? *", get label() { From 658c2f101df5162c62b3573d3b7f8deff3ef8b3c Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Wed, 23 Jul 2025 09:32:52 -0400 Subject: [PATCH 058/164] Basic page --- .../TransformSection/TransformSection.tsx | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx index ee6d00e442e6..a835804f0e33 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx @@ -126,6 +126,25 @@ function TransformSettings({ transform }: TransformSettingsProps) { } }; + const handleScheduleChange = async (schedule: string | null) => { + const { error } = await updateTransform({ + id: transform.id, + schedule, + }); + + if (error) { + sendErrorToast(t`Failed to update transform schedule`); + } else { + sendSuccessToast(t`Transform schedule updated`, async () => { + const { error } = await updateTransform({ + id: transform.id, + schedule: transform.schedule, + }); + sendUndoToast(error); + }); + } + }; + return ( @@ -172,6 +191,7 @@ function TransformSettings({ transform }: TransformSettingsProps) { value={transform.schedule} placeholder={t`Never, I'll do this manually if I need to`} clearable + onChange={handleScheduleChange} /> From 05005f91178d28fff054e607b5110a15dcbd5ddf Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Wed, 23 Jul 2025 09:39:13 -0400 Subject: [PATCH 059/164] Basic page --- .../TransformSection/TransformSection.tsx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx index a835804f0e33..69fab626189a 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx +++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformSection/TransformSection.tsx @@ -1,5 +1,6 @@ import { t } from "ttag"; +import { skipToken, useListDatabaseSchemasQuery } from "metabase/api"; import { NameDescriptionInput } from "metabase/metadata/components/NameDescriptionInput"; import { useMetadataToasts } from "metabase/metadata/hooks"; import type { TransformSectionProps } from "metabase/plugins"; @@ -23,19 +24,24 @@ import { SCHEDULE_OPTIONS } from "./constants"; export function TransformSection({ transformId }: TransformSectionProps) { const { data: transform } = useGetTransformQuery(transformId); + const databaseId = transform?.source?.query?.database; + const { data: schemas } = useListDatabaseSchemasQuery( + databaseId != null ? { id: databaseId } : skipToken, + ); - if (!transform) { + if (transform == null || schemas == null) { return null; } - return ; + return ; } type TransformSettingsProps = { transform: Transform; + schemas: string[]; }; -function TransformSettings({ transform }: TransformSettingsProps) { +function TransformSettings({ transform, schemas }: TransformSettingsProps) { const [updateTransform] = useUpdateTransformMutation(); const { sendErrorToast, sendSuccessToast, sendUndoToast } = useMetadataToasts(); @@ -177,7 +183,7 @@ function TransformSettings({ transform }: TransformSettingsProps) { + {isCustom && ( + + )} + + ); +} + +function getOptionValue(value: string | null) { + if (value == null) { + return EMPTY_OPTION.value; + } + if (SHORTCUT_OPTIONS.some((option) => option.value === value)) { + return value; + } + return CUSTOM_OPTION.value; +} diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/constants.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/ScheduleSettings/constants.ts similarity index 62% rename from enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/constants.ts rename to enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/ScheduleSettings/constants.ts index 8244a6fbbe1b..b76d615365a9 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/constants.ts +++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/ScheduleSettings/constants.ts @@ -1,6 +1,22 @@ import { t } from "ttag"; -export const SCHEDULE_OPTIONS = [ +export const DEFAULT_SCHEDULE = "0 0 * * * ? *"; + +export const EMPTY_OPTION = { + value: "none", + get label() { + return t`Never`; + }, +}; + +export const CUSTOM_OPTION = { + value: "custom", + get label() { + return t`Custom`; + }, +}; + +export const SHORTCUT_OPTIONS = [ { value: "0 0 0/1 * * ? *", get label() { @@ -38,3 +54,5 @@ export const SCHEDULE_OPTIONS = [ }, }, ]; + +export const ALL_OPTIONS = [EMPTY_OPTION, ...SHORTCUT_OPTIONS, CUSTOM_OPTION]; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/ScheduleSettings/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/ScheduleSettings/index.ts new file mode 100644 index 000000000000..6261ba859a3d --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/ScheduleSettings/index.ts @@ -0,0 +1 @@ +export * from "./ScheduleSettings"; diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/TransformSettings.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/TransformSettings.tsx index f3e2f9276a2f..9774d4d99f10 100644 --- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/TransformSettings.tsx +++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/TransformSettings/TransformSettings.tsx @@ -30,7 +30,7 @@ import { transformQueryUrl, } from "../../../utils/urls"; -import { SCHEDULE_OPTIONS } from "./constants"; +import { ScheduleSettings } from "./ScheduleSettings"; type TransformSettingsProps = { transform: Transform; @@ -250,12 +250,8 @@ export function TransformSettings({ transform }: TransformSettingsProps) { {t`Schedule`} -