Skip to content

Commit 4f854a3

Browse files
maerchmaerch
authored
Fix variant quick actions on blocks without source set (#50)
* Fix issues with asset that do not have a source set * Bump version for patch fix * Prefix image selection panel with `ly.img.ai/` so it closes with the ai panels * Bump ai plugins to 0.1.2 * Add scripts for easier publishing of ai plugins * Use demo template for ai-demo * Add `test` script in root level package.json * Fix handling of buffer urls for some cases * Bump to 0.1.3
1 parent ef626b2 commit 4f854a3

File tree

19 files changed

+261
-90
lines changed

19 files changed

+261
-90
lines changed

‎examples/web/src/pages/ai-demo.tsx‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,9 @@ function App() {
6868
instance.feature.enable('ly.img.preview', false);
6969
instance.feature.enable('ly.img.placeholder', false);
7070

71-
await instance.createVideoScene();
71+
await instance.engine.scene.loadFromArchiveURL(
72+
`https://img.ly/showcases/cesdk/cases/ai-editor/ai_editor_video.archive`
73+
);
7274

7375
const onRateLimitExceeded: RateLimitOptions<any>['onRateLimitExceeded'] =
7476
() => {
@@ -170,18 +172,18 @@ function App() {
170172
})
171173
);
172174

173-
const page = instance.engine.scene.getCurrentPage();
174-
if (page != null) {
175-
const pageFill = instance.engine.block.getFill(page);
176-
instance.engine.block.setColorRGBA(
177-
pageFill,
178-
'fill/color/value',
179-
1,
180-
1,
181-
1,
182-
1
183-
);
184-
}
175+
// const page = instance.engine.scene.getCurrentPage();
176+
// if (page != null) {
177+
// const pageFill = instance.engine.block.getFill(page);
178+
// instance.engine.block.setColorRGBA(
179+
// pageFill,
180+
// 'fill/color/value',
181+
// 1,
182+
// 1,
183+
// 1,
184+
// 1
185+
// );
186+
// }
185187
});
186188
} else if (cesdk.current != null) {
187189
cesdk.current.dispose();

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"clean": "pnpm recursive run clean",
88
"purge": "pnpm exec rimraf node_modules && pnpm recursive run purge",
99
"dev": "pnpm run --recursive --parallel dev",
10+
"test": "pnpm recursive run test",
1011
"check:all": "pnpm recursive run check:all",
1112
"check:types": "pnpm recursive run check:types"
1213
},

‎packages/plugin-ai-apps-web/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@imgly/plugin-ai-apps-web",
3-
"version": "0.1.0",
3+
"version": "0.1.3",
44
"description": "AI apps orchestration plugin for the CE.SDK editor",
55
"keywords": ["CE.SDK", "plugin", "AI", "ai apps"],
66
"repository": {

‎packages/plugin-ai-audio-generation-web/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@imgly/plugin-ai-audio-generation-web",
3-
"version": "0.1.0",
3+
"version": "0.1.3",
44
"description": "AI audio generation plugin for the CE.SDK editor",
55
"keywords": ["CE.SDK", "plugin", "AI", "audio-generation"],
66
"repository": {

‎packages/plugin-ai-generation-web/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@imgly/plugin-ai-generation-web",
3-
"version": "0.1.0",
3+
"version": "0.1.3",
44
"description": "AI generation plugin for the CE.SDK editor",
55
"keywords": ["CE.SDK", "plugin", "AI", "ai generation"],
66
"repository": {

‎packages/plugin-ai-generation-web/src/common/renderImageUrlProperty.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,10 @@ function renderImageUrlProperty(
4141
action: {
4242
label: 'Select Image',
4343
onClick: () => {
44-
cesdk?.ui.openPanel(`${provderId}.imageSelection`, {
44+
if (cesdk == null) return;
45+
46+
const panelId = getImageSelectionPanelId(provderId);
47+
cesdk.ui.openPanel(panelId, {
4548
payload: {
4649
onSelect: (assetResult: AssetResult) => {
4750
if (assetResult.meta?.uri != null) {
@@ -72,7 +75,7 @@ function createPanels(providerId: string, cesdk?: CreativeEditorSDK) {
7275

7376
cesdk.ui.registerPanel<{
7477
onSelect: (assetResult: AssetResult) => void;
75-
}>(`${providerId}.imageSelection`, ({ builder, payload }) => {
78+
}>(getImageSelectionPanelId(providerId), ({ builder, payload }) => {
7679
builder.Library(`${providerId}.library.image`, {
7780
entries: ['ly.img.image'],
7881
onSelect: async (asset) => {
@@ -88,7 +91,7 @@ function createPanels(providerId: string, cesdk?: CreativeEditorSDK) {
8891
});
8992
} else if (mimeType.startsWith('image/')) {
9093
payload?.onSelect(asset);
91-
cesdk?.ui.closePanel(`${providerId}.imageSelection`);
94+
cesdk?.ui.closePanel(getImageSelectionPanelId(providerId));
9295
} else {
9396
cesdk.ui.showNotification({
9497
type: 'warning',
@@ -100,4 +103,8 @@ function createPanels(providerId: string, cesdk?: CreativeEditorSDK) {
100103
});
101104
}
102105

106+
function getImageSelectionPanelId(providerId: string) {
107+
return `ly.img.ai/${providerId}.imageSelection`;
108+
}
109+
103110
export default renderImageUrlProperty;

‎packages/plugin-ai-generation-web/src/generation/quickAction/consumeGeneratedResult.ts‎

Lines changed: 66 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
type OutputKind
88
} from '../provider';
99
import { ApplyCallbacks } from './types';
10+
import { mimeTypeToExtension } from '@imgly/plugin-utils';
1011
import { isAsyncGenerator } from '../../utils';
1112

1213
type ConsumeGeneratedResultOptions = {
@@ -135,8 +136,17 @@ async function getApplyCallbacksForImage<O extends Output>(
135136
'fill/image/sourceSet'
136137
);
137138
const [sourceBefore] = sourceSetBefore;
139+
let uriBefore: string | undefined;
140+
if (sourceBefore == null) {
141+
uriBefore = cesdk.engine.block.getString(
142+
fillBlock,
143+
'fill/image/imageFileURI'
144+
);
145+
}
138146

139-
const mimeType = await cesdk.engine.editor.getMimeType(sourceBefore.uri);
147+
const mimeType = await cesdk.engine.editor.getMimeType(
148+
sourceBefore?.uri ?? uriBefore
149+
);
140150
abortSignal.throwIfAborted();
141151

142152
if (mimeType === 'image/svg+xml') {
@@ -173,34 +183,68 @@ async function getApplyCallbacksForImage<O extends Output>(
173183

174184
const uri = await reuploadImage(cesdk, url, generatedMimeType);
175185

176-
const sourceSetAfter = [
177-
{
178-
uri,
179-
width: sourceBefore.width,
180-
height: sourceBefore.height
181-
}
182-
];
183-
cesdk.engine.block.setSourceSet(
184-
fillBlock,
185-
'fill/image/sourceSet',
186-
sourceSetAfter
187-
);
188-
applyCrop();
186+
const sourceSetAfter = sourceBefore
187+
? [
188+
{
189+
uri,
190+
width: sourceBefore.width,
191+
height: sourceBefore.height
192+
}
193+
]
194+
: undefined;
195+
const uriAfter = uri;
189196

190-
const onBefore = () => {
191-
cesdk.engine.block.setSourceSet(
197+
if (sourceSetAfter == null) {
198+
cesdk.engine.block.setString(
192199
fillBlock,
193-
'fill/image/sourceSet',
194-
sourceSetBefore
200+
'fill/image/imageFileURI',
201+
uriAfter
195202
);
196-
applyCrop();
197-
};
198-
const onAfter = () => {
203+
} else {
199204
cesdk.engine.block.setSourceSet(
200205
fillBlock,
201206
'fill/image/sourceSet',
202207
sourceSetAfter
203208
);
209+
}
210+
applyCrop();
211+
212+
const onBefore = () => {
213+
if (sourceSetBefore == null) {
214+
if (uriBefore == null) {
215+
throw new Error('No image URI found');
216+
}
217+
cesdk.engine.block.setString(
218+
fillBlock,
219+
'fill/image/imageFileURI',
220+
uriBefore
221+
);
222+
} else {
223+
cesdk.engine.block.setSourceSet(
224+
fillBlock,
225+
'fill/image/sourceSet',
226+
sourceSetBefore
227+
);
228+
}
229+
applyCrop();
230+
};
231+
const onAfter = () => {
232+
if (sourceSetAfter == null) {
233+
if (uriAfter == null) {
234+
throw new Error('No image URI found');
235+
}
236+
cesdk.engine.block.setString(
237+
fillBlock,
238+
'fill/image/imageFileURI',
239+
uriAfter
240+
);
241+
} else {
242+
cesdk.engine.block.setSourceSet(
243+
fillBlock,
244+
'fill/image/sourceSet',
245+
sourceSetAfter
246+
);
247+
}
204248
applyCrop();
205249
};
206250
const onCancel = () => {
@@ -246,7 +290,7 @@ async function reuploadImage(
246290
): Promise<string> {
247291
const response = await fetch(url);
248292
const blob = await response.blob();
249-
const file = new File([blob], `image.${getFileExtension(mimeType)}`, {
293+
const file = new File([blob], `image.${mimeTypeToExtension(mimeType)}`, {
250294
type: mimeType
251295
});
252296
const assetDefinition = await cesdk.unstable_upload(file, () => {});
@@ -257,15 +301,4 @@ async function reuploadImage(
257301
return url;
258302
}
259303

260-
function getFileExtension(mimeType: string): string {
261-
const mimeTypeToExtension: Record<string, string> = {
262-
'image/png': 'png',
263-
'image/jpeg': 'jpg',
264-
'image/webp': 'webp',
265-
'image/gif': 'gif',
266-
'image/svg+xml': 'svg'
267-
};
268-
return mimeTypeToExtension[mimeType] ?? 'png';
269-
}
270-
271304
export default consumeGeneratedResult;

‎packages/plugin-ai-image-generation-web/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@imgly/plugin-ai-image-generation-web",
3-
"version": "0.1.0",
3+
"version": "0.1.3",
44
"description": "AI image generation plugin for the CE.SDK editor",
55
"keywords": ["CE.SDK", "plugin", "AI", "image-generation"],
66
"repository": {

‎packages/plugin-ai-image-generation-web/src/fal-ai/GeminiFlashEdit.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ function getProvider(
6464
middleware: config.middleware,
6565
getBlockInput: async (input) => {
6666
const { width, height } = await getImageDimensionsFromURL(
67-
input.image_url
67+
input.image_url,
68+
cesdk
6869
);
6970
return Promise.resolve({
7071
image: {

‎packages/plugin-ai-image-generation-web/src/fal-ai/GeminiFlashEditQuickActions.ts‎

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { bufferURIToObjectURL } from '@imgly/plugin-utils';
12
import { QuickAction, Output } from '@imgly/plugin-ai-generation-web';
23
import CreativeEditorSDK from '@cesdk/cesdk-js';
34

@@ -80,24 +81,31 @@ function createBaseQuickAction<I, O extends Output>(
8081
}
8182

8283
// Helper to get image source from a block
83-
async function getImageSource(blockId: number, cesdk: CreativeEditorSDK) {
84+
async function getImageUri(
85+
blockId: number,
86+
cesdk: CreativeEditorSDK
87+
): Promise<string> {
88+
let uri;
8489
const fillBlock = cesdk.engine.block.getFill(blockId);
8590
const sourceSet = cesdk.engine.block.getSourceSet(
8691
fillBlock,
8792
'fill/image/sourceSet'
8893
);
8994
const [source] = sourceSet;
9095
if (source == null) {
91-
throw new Error('No image source found');
96+
uri = cesdk.engine.block.getString(fillBlock, 'fill/image/imageFileURI');
97+
if (uri == null) throw new Error('No image source/uri found');
98+
} else {
99+
uri = source.uri;
92100
}
93101

94102
// Check if the image is SVG (not supported)
95-
const mimeType = await cesdk.engine.editor.getMimeType(source.uri);
103+
const mimeType = await cesdk.engine.editor.getMimeType(uri);
96104
if (mimeType === 'image/svg+xml') {
97105
throw new Error('SVG images are not supported');
98106
}
99107

100-
return source;
108+
return bufferURIToObjectURL(uri, cesdk);
101109
}
102110

103111
// Change Image Quick Action (with text input)
@@ -150,11 +158,11 @@ function ChangeImageQuickAction(
150158
if (!prompt) return;
151159

152160
const [blockId] = cesdk.engine.block.findAllSelected();
153-
const source = await getImageSource(blockId, cesdk);
161+
const uri = await getImageUri(blockId, cesdk);
154162

155163
generate({
156164
prompt,
157-
image_url: source.uri,
165+
image_url: uri,
158166
blockId
159167
});
160168

@@ -284,13 +292,13 @@ function CreateVariantQuickAction(
284292
);
285293

286294
// Get the source of the duplicated block
287-
const source = await getImageSource(duplicated, cesdk);
295+
const uri = await getImageUri(duplicated, cesdk);
288296

289297
// Generate a variant for the duplicated block
290298
generate(
291299
{
292300
prompt,
293-
image_url: source.uri,
301+
image_url: uri,
294302
blockId: duplicated
295303
},
296304
{
@@ -327,7 +335,7 @@ function CreateVideoQuickAction(
327335
onClick: async () => {
328336
try {
329337
const [blockId] = cesdk.engine.block.findAllSelected();
330-
const source = await getImageSource(blockId, cesdk);
338+
const uri = await getImageUri(blockId, cesdk);
331339

332340
cesdk.ui.openPanel('ly.img.ai/video-generation');
333341
cesdk.ui.experimental.setGlobalStateValue(
@@ -336,7 +344,7 @@ function CreateVideoQuickAction(
336344
);
337345
cesdk.ui.experimental.setGlobalStateValue(
338346
'fal-ai/minimax/video-01-live/image-to-video.image_url',
339-
source.uri
347+
uri
340348
);
341349

342350
closeMenu();
@@ -427,11 +435,11 @@ function StyleTransferQuickAction(
427435
closeMenu();
428436
const [blockId] =
429437
cesdk.engine.block.findAllSelected();
430-
const source = await getImageSource(blockId, cesdk);
438+
const uri = await getImageUri(blockId, cesdk);
431439

432440
generate({
433441
prompt: style.prompt,
434-
image_url: source.uri,
442+
image_url: uri,
435443
blockId
436444
});
437445
} catch (error) {
@@ -560,11 +568,11 @@ function ArtistStyleQuickAction(
560568
closeMenu();
561569
const [blockId] =
562570
cesdk.engine.block.findAllSelected();
563-
const source = await getImageSource(blockId, cesdk);
571+
const uri = await getImageUri(blockId, cesdk);
564572

565573
generate({
566574
prompt: artist.prompt,
567-
image_url: source.uri,
575+
image_url: uri,
568576
blockId
569577
});
570578
} catch (error) {

0 commit comments

Comments
 (0)