| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750 |
- import { PermissionAction } from '@supabase/shared-types/out/constants'
- import { useParams } from 'common'
- import { Eye, EyeOff, Loader2 } from 'lucide-react'
- import { useState } from 'react'
- import type { UseFormReturn } from 'react-hook-form'
- import {
- Button,
- FormControl,
- FormField,
- Input,
- Select,
- SelectContent,
- SelectGroup,
- SelectItem,
- SelectSeparator,
- SelectTrigger,
- TextArea,
- WarningIcon,
- } from 'ui'
- import { Admonition } from 'ui-patterns'
- import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input'
- import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
- import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
- import type { DestinationPanelSchemaType } from './DestinationForm.schema'
- import { InlineLink } from '@/components/ui/InlineLink'
- import { getKeys, useAPIKeysQuery } from '@/data/api-keys/api-keys-query'
- import { useAnalyticsBucketsQuery } from '@/data/storage/analytics-buckets-query'
- import { useIcebergNamespacesQuery } from '@/data/storage/iceberg-namespaces-query'
- import { useStorageCredentialsQuery } from '@/data/storage/s3-access-key-query'
- import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions'
- export const BigQueryFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
- return (
- <div className="flex flex-col gap-y-6 p-5">
- <p className="text-sm font-medium text-foreground">BigQuery settings</p>
- <div className="flex flex-col gap-y-4">
- <FormField
- control={form.control}
- name="projectId"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="Project ID"
- description="The Google Cloud project ID where data will be sent"
- >
- <FormControl>
- <Input {...field} placeholder="my-gcp-project" />
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="datasetId"
- render={({ field }) => (
- <FormItemLayout
- label="Dataset ID"
- layout="horizontal"
- description="The BigQuery dataset where replicated tables will be created"
- >
- <FormControl>
- <Input {...field} placeholder="my_dataset" />
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="serviceAccountKey"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="Service Account Key"
- description="Service account credentials JSON for authenticating with BigQuery"
- >
- <FormControl>
- <TextArea
- {...field}
- rows={5}
- maxLength={5000}
- placeholder='{"type": "service_account", "project_id": "...", ...}'
- className="font-mono text-xs"
- />
- </FormControl>
- </FormItemLayout>
- )}
- />
- </div>
- </div>
- )
- }
- export const DuckLakeFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
- const [showCatalogUrl, setShowCatalogUrl] = useState(false)
- const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)
- return (
- <div className="flex flex-col gap-y-6 p-5">
- <p className="text-sm font-medium text-foreground">DuckLake settings</p>
- <div className="flex flex-col gap-y-1">
- <p className="text-sm font-medium text-foreground">Catalog</p>
- <p className="text-sm text-foreground-light">
- Configure the PostgreSQL-backed DuckLake catalog and the S3-compatible storage location
- for replicated data.
- </p>
- </div>
- <div className="flex flex-col gap-y-4">
- <FormField
- control={form.control}
- name="ducklakeCatalogUrl"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="Catalog URL"
- description="A PostgreSQL connection string for the DuckLake catalog"
- >
- <FormControl>
- <PasswordInput
- value={field.value ?? ''}
- type={showCatalogUrl ? 'text' : 'password'}
- placeholder="postgres://user:pass@host:5432/ducklake_catalog"
- onChange={(event) => field.onChange(event.target.value)}
- actions={
- <div className="flex items-center justify-center">
- <Button
- type="default"
- className="w-7"
- icon={showCatalogUrl ? <Eye /> : <EyeOff />}
- onClick={() => setShowCatalogUrl(!showCatalogUrl)}
- />
- </div>
- }
- />
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="ducklakeDataPath"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="Data path"
- description="An S3 path where DuckLake data files will be written"
- >
- <FormControl>
- <Input {...field} placeholder="s3://bucket/path" value={field.value ?? ''} />
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="ducklakePoolSize"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="Pool size"
- description="Optional number of concurrent DuckDB connections to use"
- >
- <FormControl>
- <Input
- type="number"
- min={1}
- max={6}
- value={field.value ?? ''}
- placeholder="Default: 4"
- onChange={(event) =>
- field.onChange(
- event.target.value === '' ? undefined : Number(event.target.value)
- )
- }
- />
- </FormControl>
- </FormItemLayout>
- )}
- />
- </div>
- <div className="flex flex-col gap-y-1">
- <p className="text-sm font-medium text-foreground">Object storage</p>
- <p className="text-sm text-foreground-light">
- Optional credentials and endpoint settings for S3-compatible storage providers.
- </p>
- </div>
- <div className="flex flex-col gap-y-4">
- <FormField
- control={form.control}
- name="ducklakeS3AccessKeyId"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="S3 Access Key ID"
- description="Required access key ID for the object storage provider"
- >
- <FormControl>
- <Input {...field} placeholder="my-access-key" value={field.value ?? ''} />
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="ducklakeS3SecretAccessKey"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="S3 Secret Access Key"
- description="Required secret access key for the object storage provider"
- className="relative"
- >
- <FormControl>
- <Input
- {...field}
- type={showSecretAccessKey ? 'text' : 'password'}
- placeholder="my-secret-key"
- value={field.value ?? ''}
- />
- </FormControl>
- <Button
- type="default"
- icon={showSecretAccessKey ? <Eye /> : <EyeOff />}
- className="w-7 absolute right-6 top-[4px]"
- onClick={() => setShowSecretAccessKey(!showSecretAccessKey)}
- />
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="ducklakeS3Region"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="S3 Region"
- description="Required region for the object storage provider"
- >
- <FormControl>
- <Input {...field} placeholder="us-east-1" value={field.value ?? ''} />
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="ducklakeS3Endpoint"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="S3 Endpoint"
- description="Required endpoint without the protocol scheme, for example `127.0.0.1:5000/s3`"
- >
- <FormControl>
- <Input {...field} placeholder="127.0.0.1:5000/s3" value={field.value ?? ''} />
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="ducklakeS3UrlStyle"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="S3 URL style"
- description="Choose `path` for MinIO/Briven-style endpoints or `vhost` for AWS-style virtual host addressing"
- >
- <FormControl>
- <Select value={field.value ?? 'path'} onValueChange={field.onChange}>
- <SelectTrigger>{field.value ?? 'path'}</SelectTrigger>
- <SelectContent>
- <SelectItem value="path">path</SelectItem>
- <SelectItem value="vhost">vhost</SelectItem>
- </SelectContent>
- </Select>
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="ducklakeS3UseSsl"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="Use SSL"
- description="Whether to use SSL when connecting to the S3-compatible endpoint"
- >
- <FormControl>
- <Select
- value={field.value === false ? 'false' : 'true'}
- onValueChange={(value) => field.onChange(value === 'true')}
- >
- <SelectTrigger>{field.value === false ? 'false' : 'true'}</SelectTrigger>
- <SelectContent>
- <SelectItem value="true">true</SelectItem>
- <SelectItem value="false">false</SelectItem>
- </SelectContent>
- </Select>
- </FormControl>
- </FormItemLayout>
- )}
- />
- </div>
- <div className="flex flex-col gap-y-1">
- <p className="text-sm font-medium text-foreground">Maintenance</p>
- <p className="text-sm text-foreground-light">
- Optional settings for DuckLake metadata tables and snapshot cleanup.
- </p>
- </div>
- <div className="flex flex-col gap-y-4">
- <FormField
- control={form.control}
- name="ducklakeMetadataSchema"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="Metadata schema"
- description="Schema used for DuckLake metadata tables in PostgreSQL"
- >
- <FormControl>
- <Input {...field} placeholder="ducklake" value={field.value ?? ''} />
- </FormControl>
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="ducklakeExpireSnapshotsOlderThan"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="Expire snapshots older than"
- description="Optional snapshot retention interval, for example `7 days`"
- >
- <FormControl>
- <Input {...field} placeholder="7 days" value={field.value ?? ''} />
- </FormControl>
- </FormItemLayout>
- )}
- />
- </div>
- </div>
- )
- }
- /**
- * [Joshen] JFYI I'd foresee a possible UX friction point here regarding S3 access key IDs and secret access keys
- * - We'd allow users to select access key IDs via a dropdown here, but require a text input for secret access keys
- * - Chances are most users wouldn't have the corresponding secret access key for the selected key ID at the top of their heads
- * - So highly likely may have to default to "Create a new key" -> which from here they won't know the secret access key thereafter
- * - And it'll end up just creating more keys for each destination
- * Ideal scenario: Just select an access key ID, we then apply the secret access key in the PATCH request, so FE has no
- * context of the secret access key at any point
- */
- export const AnalyticsBucketFields = ({
- form,
- setIsFormInteracting,
- onSelectNewBucket,
- }: {
- form: UseFormReturn<DestinationPanelSchemaType>
- setIsFormInteracting: (value: boolean) => void
- onSelectNewBucket: () => void
- }) => {
- const { warehouseName, s3AccessKeyId, namespace } = form.watch()
- const [showCatalogToken, setShowCatalogToken] = useState(false)
- const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)
- const { ref: projectRef } = useParams()
- const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
- const { data: apiKeys } = useAPIKeysQuery(
- { projectRef, reveal: true },
- { enabled: canReadAPIKeys }
- )
- const { serviceKey } = getKeys(apiKeys)
- const serviceApiKey = serviceKey?.api_key ?? ''
- const {
- data: keysData,
- isSuccess: isSuccessKeys,
- isPending: isLoadingKeys,
- isError: isErrorKeys,
- } = useStorageCredentialsQuery({ projectRef })
- const s3Keys = keysData?.data ?? []
- const keyNoLongerExists =
- (s3AccessKeyId ?? '').length > 0 &&
- s3AccessKeyId !== CREATE_NEW_KEY &&
- !s3Keys.find((k) => k.access_key === s3AccessKeyId)
- const {
- data: analyticsBuckets = [],
- isPending: isLoadingBuckets,
- isError: isErrorBuckets,
- } = useAnalyticsBucketsQuery({ projectRef })
- const canSelectNamespace = !!warehouseName && !!serviceApiKey
- const {
- data: namespaces = [],
- isPending: isLoadingNamespaces,
- isError: isErrorNamespaces,
- } = useIcebergNamespacesQuery(
- { projectRef, warehouse: warehouseName },
- { enabled: !!serviceApiKey }
- )
- return (
- <div className="flex flex-col gap-y-6 p-5">
- <p className="text-sm font-medium text-foreground">Analytics Bucket settings</p>
- <div className="flex flex-col gap-y-4">
- <FormField
- control={form.control}
- name="warehouseName"
- render={({ field }) => (
- <FormItemLayout
- label="Bucket"
- layout="horizontal"
- description="The Analytics Bucket where data will be stored"
- >
- {isLoadingBuckets ? (
- <Button
- disabled
- type="default"
- className="w-full justify-between"
- size="small"
- iconRight={<Loader2 className="animate-spin" />}
- >
- Retrieving buckets
- </Button>
- ) : isErrorBuckets ? (
- <Button
- disabled
- type="default"
- className="w-full justify-start"
- size="small"
- icon={<WarningIcon />}
- >
- Failed to retrieve buckets
- </Button>
- ) : (
- <FormControl>
- <Select
- value={field.value}
- onValueChange={(value) => {
- if (value === 'new-bucket') {
- onSelectNewBucket()
- } else {
- setIsFormInteracting(true)
- field.onChange(value)
- // [Joshen] Ideally should select the first namespace of the selected bucket
- form.setValue('namespace', '')
- }
- }}
- >
- <SelectTrigger>{field.value || 'Select a bucket'}</SelectTrigger>
- <SelectContent>
- <SelectGroup>
- {analyticsBuckets.length === 0 ? (
- <SelectItem value="__no_buckets__" disabled>
- No buckets available
- </SelectItem>
- ) : (
- analyticsBuckets.map((bucket) => (
- <SelectItem key={bucket.name} value={bucket.name}>
- {bucket.name}
- </SelectItem>
- ))
- )}
- <SelectSeparator />
- <SelectItem value="new-bucket">Create a new bucket</SelectItem>
- </SelectGroup>
- </SelectContent>
- </Select>
- </FormControl>
- )}
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="namespace"
- render={({ field }) => (
- <FormItemLayout
- label="Namespace"
- layout="horizontal"
- description="The namespace within the bucket where tables will be organized"
- >
- {isLoadingNamespaces && canSelectNamespace ? (
- <Button
- disabled
- type="default"
- className="w-full justify-between"
- size="small"
- iconRight={<Loader2 className="animate-spin" />}
- >
- Retrieving namespaces
- </Button>
- ) : isErrorNamespaces ? (
- <Button
- disabled
- type="default"
- className="w-full justify-start"
- size="small"
- icon={<WarningIcon />}
- >
- Failed to retrieve namespaces
- </Button>
- ) : (
- <FormControl>
- <Select
- value={field.value}
- onValueChange={(value) => {
- setIsFormInteracting(true)
- field.onChange(value)
- }}
- disabled={!canSelectNamespace}
- >
- <SelectTrigger>
- {!canSelectNamespace
- ? 'Select a warehouse first'
- : field.value === CREATE_NEW_NAMESPACE
- ? 'Create a new namespace'
- : field.value || 'Select a namespace'}
- </SelectTrigger>
- <SelectContent>
- <SelectGroup>
- {namespaces.length === 0 ? (
- <SelectItem value="__no_namespaces__" disabled>
- No namespaces available
- </SelectItem>
- ) : (
- namespaces.map((namespace) => (
- <SelectItem key={namespace} value={namespace}>
- {namespace}
- </SelectItem>
- ))
- )}
- <SelectSeparator />
- <SelectItem key={CREATE_NEW_NAMESPACE} value={CREATE_NEW_NAMESPACE}>
- Create a new namespace
- </SelectItem>
- </SelectGroup>
- </SelectContent>
- </Select>
- </FormControl>
- )}
- </FormItemLayout>
- )}
- />
- {namespace === CREATE_NEW_NAMESPACE && (
- <FormField
- control={form.control}
- name="newNamespaceName"
- render={({ field }) => (
- <FormItemLayout
- label="New Namespace Name"
- layout="horizontal"
- description="A unique name for the new namespace"
- >
- <FormControl>
- <Input {...field} placeholder="new_namespace" value={field.value || ''} />
- </FormControl>
- </FormItemLayout>
- )}
- />
- )}
- <FormField
- control={form.control}
- name="catalogToken"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="Catalog Token"
- description={
- <>
- Automatically retrieved from your project's{' '}
- <InlineLink href={`/project/${projectRef}/settings/api-keys`}>
- service role key
- </InlineLink>
- </>
- }
- >
- <PasswordInput
- disabled
- value={field.value}
- type={showCatalogToken ? 'text' : 'password'}
- placeholder="Auto-populated"
- actions={
- serviceApiKey ? (
- <div className="flex items-center justify-center">
- <Button
- type="default"
- className="w-7"
- icon={showCatalogToken ? <Eye /> : <EyeOff />}
- onClick={() => setShowCatalogToken(!showCatalogToken)}
- />
- </div>
- ) : null
- }
- />
- </FormItemLayout>
- )}
- />
- <FormField
- control={form.control}
- name="s3AccessKeyId"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="S3 Access Key ID"
- description={
- <div className="flex flex-col gap-y-2">
- <p>
- Access keys are managed in your Storage{' '}
- <InlineLink href={`/project/${projectRef}/storage/s3`}>S3 settings</InlineLink>
- </p>
- {isSuccessKeys && keyNoLongerExists && (
- <Admonition
- type="warning"
- title="Unable to find access key ID in project"
- description={
- <>
- Please select another key or create a new set, as this destination will
- not work otherwise. S3 access keys can be managed in your{' '}
- <InlineLink href={`/project/${projectRef}/storage/files/settings`}>
- storage settings
- </InlineLink>
- .
- </>
- }
- />
- )}
- {s3AccessKeyId === CREATE_NEW_KEY && (
- <Admonition
- type="default"
- description="A new set of S3 access keys will be created."
- />
- )}
- </div>
- }
- >
- {isLoadingKeys ? (
- <Button
- disabled
- type="default"
- className="w-full justify-between"
- size="small"
- iconRight={<Loader2 className="animate-spin" />}
- >
- Retrieving keys
- </Button>
- ) : isErrorKeys ? (
- <Button
- disabled
- type="default"
- className="w-full justify-start"
- size="small"
- icon={<WarningIcon />}
- >
- Failed to retrieve keys
- </Button>
- ) : (
- <FormControl>
- <Select value={field.value} onValueChange={field.onChange}>
- <SelectTrigger>
- {field.value === CREATE_NEW_KEY
- ? 'Create a new key'
- : (field.value ?? '').length === 0
- ? 'Select an access key ID'
- : field.value}
- </SelectTrigger>
- <SelectContent>
- <SelectGroup>
- {s3Keys.map((key) => (
- <SelectItem key={key.id} value={key.access_key}>
- {key.access_key}
- <p className="text-foreground-lighter">{key.description}</p>
- </SelectItem>
- ))}
- <SelectSeparator />
- <SelectItem key={CREATE_NEW_KEY} value={CREATE_NEW_KEY}>
- Create a new key
- </SelectItem>
- </SelectGroup>
- </SelectContent>
- </Select>
- </FormControl>
- )}
- </FormItemLayout>
- )}
- />
- {s3AccessKeyId !== CREATE_NEW_KEY && (
- <FormField
- control={form.control}
- name="s3SecretAccessKey"
- render={({ field }) => (
- <FormItemLayout
- layout="horizontal"
- label="S3 Secret Access Key"
- className="relative"
- description="The secret key corresponding to your selected access key ID."
- >
- <FormControl>
- <Input
- {...field}
- type={showSecretAccessKey ? 'text' : 'password'}
- value={field.value ?? ''}
- placeholder="Provide the secret access key"
- />
- </FormControl>
- <Button
- type="default"
- icon={showSecretAccessKey ? <Eye /> : <EyeOff />}
- className="w-7 absolute right-6 top-[4px]"
- onClick={() => setShowSecretAccessKey(!showSecretAccessKey)}
- />
- </FormItemLayout>
- )}
- />
- )}
- </div>
- </div>
- )
- }
|