Skip to content

Commit a45d950

Browse files
mudlermudler
committed
feat(agent-jobs): add multimedia support
Signed-off-by: Ettore Di Giacinto <[email protected]>
1 parent 54b5dfa commit a45d950

File tree

10 files changed

+949
-45
lines changed

10 files changed

+949
-45
lines changed

‎core/http/endpoints/localai/agent_jobs.go‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,28 @@ func ExecuteJobEndpoint(app *application.Application) echo.HandlerFunc {
147147
req.Parameters = make(map[string]string)
148148
}
149149

150-
jobID, err := app.AgentJobService().ExecuteJob(req.TaskID, req.Parameters, "api")
150+
// Build multimedia struct from request
151+
var multimedia *struct {
152+
Images []string
153+
Videos []string
154+
Audios []string
155+
Files []string
156+
}
157+
if len(req.Images) > 0 || len(req.Videos) > 0 || len(req.Audios) > 0 || len(req.Files) > 0 {
158+
multimedia = &struct {
159+
Images []string
160+
Videos []string
161+
Audios []string
162+
Files []string
163+
}{
164+
Images: req.Images,
165+
Videos: req.Videos,
166+
Audios: req.Audios,
167+
Files: req.Files,
168+
}
169+
}
170+
171+
jobID, err := app.AgentJobService().ExecuteJob(req.TaskID, req.Parameters, "api", multimedia)
151172
if err != nil {
152173
return c.JSON(http.StatusBadRequest, map[string]string{"error": err.Error()})
153174
}
@@ -323,7 +344,7 @@ func ExecuteTaskByNameEndpoint(app *application.Application) echo.HandlerFunc {
323344
return c.JSON(http.StatusNotFound, map[string]string{"error": "Task not found: " + name})
324345
}
325346

326-
jobID, err := app.AgentJobService().ExecuteJob(task.ID, params, "api")
347+
jobID, err := app.AgentJobService().ExecuteJob(task.ID, params, "api", nil)
327348
if err != nil {
328349
return c.JSON(http.StatusBadRequest, map[string]string{"error": err.Error()})
329350
}

‎core/http/views/agent-jobs.html‎

Lines changed: 134 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -365,21 +365,40 @@ <h2 class="text-2xl font-semibold text-[var(--color-text-primary)]">Job History<
365365
x-cloak
366366
@click.away="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''"
367367
class="fixed inset-0 bg-black/50 flex items-center justify-center z-50">
368-
<div class="bg-[var(--color-bg-secondary)] border border-[var(--color-primary-border)]/20 rounded-xl p-8 max-w-2xl w-full mx-4">
369-
<div class="flex justify-between items-center mb-6">
368+
<div class="bg-[var(--color-bg-secondary)] border border-[var(--color-primary-border)]/20 rounded-xl max-w-2xl w-full mx-4 max-h-[90vh] flex flex-col">
369+
<div class="flex justify-between items-center p-8 pb-6 border-b border-[var(--color-primary-border)]/20">
370370
<h3 class="text-2xl font-semibold text-[var(--color-text-primary)]">Execute Task</h3>
371-
<button @click="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''"
371+
<button @click="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''; executionMultimedia = {images: '', videos: '', audios: '', files: ''}; executeModalTab = 'parameters'"
372372
class="text-[var(--color-text-secondary)] hover:text-[var(--color-text-primary)]">
373373
<i class="fas fa-times text-xl"></i>
374374
</button>
375375
</div>
376376
<template x-if="selectedTaskForExecution">
377-
<div class="space-y-4">
377+
<div class="flex flex-col flex-1 min-h-0">
378+
<div class="flex-1 overflow-y-auto px-8 py-6 space-y-4">
378379
<div>
379380
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Task</label>
380381
<div class="text-[var(--color-text-secondary)]" x-text="selectedTaskForExecution.name"></div>
381382
</div>
382-
<div>
383+
384+
<!-- Tabs for Parameters and Multimedia -->
385+
<div class="border-b border-[var(--color-primary-border)]/20">
386+
<div class="flex space-x-4">
387+
<button @click="executeModalTab = 'parameters'"
388+
:class="executeModalTab === 'parameters' ? 'border-b-2 border-[var(--color-primary)] text-[var(--color-primary)]' : 'text-[var(--color-text-secondary)] hover:text-[var(--color-text-primary)]'"
389+
class="px-4 py-2 font-medium transition-colors">
390+
Parameters
391+
</button>
392+
<button @click="executeModalTab = 'multimedia'"
393+
:class="executeModalTab === 'multimedia' ? 'border-b-2 border-[var(--color-primary)] text-[var(--color-primary)]' : 'text-[var(--color-text-secondary)] hover:text-[var(--color-text-primary)]'"
394+
class="px-4 py-2 font-medium transition-colors">
395+
Multimedia
396+
</button>
397+
</div>
398+
</div>
399+
400+
<!-- Parameters Tab -->
401+
<div x-show="executeModalTab === 'parameters'">
383402
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Parameters</label>
384403
<p class="text-xs text-[var(--color-text-secondary)] mb-3">
385404
Enter parameters as key-value pairs (one per line, format: key=value).
@@ -393,8 +412,61 @@ <h3 class="text-2xl font-semibold text-[var(--color-text-primary)]">Execute Task
393412
Example: <code class="bg-[var(--color-bg-primary)] px-1 py-0.5 rounded text-[var(--color-primary)]">user_name=Alice</code>
394413
</p>
395414
</div>
396-
<div class="flex justify-end space-x-4">
397-
<button @click="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''"
415+
416+
<!-- Multimedia Tab -->
417+
<div x-show="executeModalTab === 'multimedia'" class="space-y-4">
418+
<p class="text-xs text-[var(--color-text-secondary)] mb-3">
419+
Provide multimedia content as URLs or base64-encoded data URIs. You can also upload files which will be converted to base64.
420+
</p>
421+
422+
<!-- Images -->
423+
<div>
424+
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Images</label>
425+
<textarea x-model="executionMultimedia.images"
426+
rows="3"
427+
placeholder="https://example.com/image.png&#10;data:image/png;base64,iVBORw0KG..."
428+
class="w-full bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded px-4 py-2 text-[var(--color-text-primary)] font-mono text-sm focus:border-[var(--color-primary-border)] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
429+
<input type="file" @change="handleFileUpload($event, 'image')" accept="image/*" multiple
430+
class="mt-2 text-sm text-[var(--color-text-secondary)] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[var(--color-primary)] file:text-white hover:file:bg-[var(--color-primary-hover)]">
431+
</div>
432+
433+
<!-- Videos -->
434+
<div>
435+
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Videos</label>
436+
<textarea x-model="executionMultimedia.videos"
437+
rows="3"
438+
placeholder="https://example.com/video.mp4&#10;data:video/mp4;base64,..."
439+
class="w-full bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded px-4 py-2 text-[var(--color-text-primary)] font-mono text-sm focus:border-[var(--color-primary-border)] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
440+
<input type="file" @change="handleFileUpload($event, 'video')" accept="video/*" multiple
441+
class="mt-2 text-sm text-[var(--color-text-secondary)] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[var(--color-primary)] file:text-white hover:file:bg-[var(--color-primary-hover)]">
442+
</div>
443+
444+
<!-- Audios -->
445+
<div>
446+
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Audios</label>
447+
<textarea x-model="executionMultimedia.audios"
448+
rows="3"
449+
placeholder="https://example.com/audio.mp3&#10;data:audio/mpeg;base64,..."
450+
class="w-full bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded px-4 py-2 text-[var(--color-text-primary)] font-mono text-sm focus:border-[var(--color-primary-border)] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
451+
<input type="file" @change="handleFileUpload($event, 'audio')" accept="audio/*" multiple
452+
class="mt-2 text-sm text-[var(--color-text-secondary)] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[var(--color-primary)] file:text-white hover:file:bg-[var(--color-primary-hover)]">
453+
</div>
454+
455+
<!-- Files -->
456+
<div>
457+
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Files</label>
458+
<textarea x-model="executionMultimedia.files"
459+
rows="3"
460+
placeholder="https://example.com/file.pdf&#10;data:application/pdf;base64,..."
461+
class="w-full bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded px-4 py-2 text-[var(--color-text-primary)] font-mono text-sm focus:border-[var(--color-primary-border)] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
462+
<input type="file" @change="handleFileUpload($event, 'file')" multiple
463+
class="mt-2 text-sm text-[var(--color-text-secondary)] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[var(--color-primary)] file:text-white hover:file:bg-[var(--color-primary-hover)]">
464+
</div>
465+
</div>
466+
</div>
467+
468+
<div class="flex justify-end space-x-4 p-8 pt-6 border-t border-[var(--color-primary-border)]/20 bg-[var(--color-bg-secondary)]">
469+
<button @click="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''; executionMultimedia = {images: '', videos: '', audios: '', files: ''}; executeModalTab = 'parameters'"
398470
class="px-4 py-2 bg-[var(--color-bg-primary)] hover:bg-[#0A0E1A] text-[var(--color-text-primary)] rounded-lg transition-colors">
399471
Cancel
400472
</button>
@@ -433,6 +505,13 @@ <h3 class="text-2xl font-semibold text-[var(--color-text-primary)]">Execute Task
433505
selectedTaskForExecution: null,
434506
executionParameters: {},
435507
executionParametersText: '',
508+
executionMultimedia: {
509+
images: '',
510+
videos: '',
511+
audios: '',
512+
files: ''
513+
},
514+
executeModalTab: 'parameters',
436515
modelsConfig: [],
437516
hasModels: false,
438517
hasMCPModels: false,
@@ -528,20 +607,36 @@ <h3 class="text-2xl font-semibold text-[var(--color-text-primary)]">Execute Task
528607
// Parse parameters from text
529608
this.executionParameters = this.parseParameters(this.executionParametersText);
530609

610+
// Parse multimedia from text (split by newlines, filter empty)
611+
const parseMultimedia = (text) => {
612+
if (!text || !text.trim()) return [];
613+
return text.split('\n')
614+
.map(line => line.trim())
615+
.filter(line => line.length > 0);
616+
};
617+
618+
const requestBody = {
619+
task_id: this.selectedTaskForExecution.id,
620+
parameters: this.executionParameters,
621+
images: parseMultimedia(this.executionMultimedia.images),
622+
videos: parseMultimedia(this.executionMultimedia.videos),
623+
audios: parseMultimedia(this.executionMultimedia.audios),
624+
files: parseMultimedia(this.executionMultimedia.files)
625+
};
626+
531627
try {
532628
const response = await fetch('/api/agent/jobs/execute', {
533629
method: 'POST',
534630
headers: { 'Content-Type': 'application/json' },
535-
body: JSON.stringify({
536-
task_id: this.selectedTaskForExecution.id,
537-
parameters: this.executionParameters
538-
})
631+
body: JSON.stringify(requestBody)
539632
});
540633
if (response.ok) {
541634
this.showExecuteTaskModal = false;
542635
this.selectedTaskForExecution = null;
543636
this.executionParameters = {};
544637
this.executionParametersText = '';
638+
this.executionMultimedia = {images: '', videos: '', audios: '', files: ''};
639+
this.executeModalTab = 'parameters';
545640
this.fetchJobs();
546641
} else {
547642
const error = await response.json();
@@ -552,6 +647,34 @@ <h3 class="text-2xl font-semibold text-[var(--color-text-primary)]">Execute Task
552647
alert('Failed to execute task: ' + error.message);
553648
}
554649
},
650+
651+
handleFileUpload(event, type) {
652+
const files = event.target.files;
653+
if (!files || files.length === 0) return;
654+
655+
const dataURIs = [];
656+
let processed = 0;
657+
658+
for (let i = 0; i < files.length; i++) {
659+
const file = files[i];
660+
const reader = new FileReader();
661+
662+
reader.onload = (e) => {
663+
const dataURI = e.target.result;
664+
dataURIs.push(dataURI);
665+
processed++;
666+
667+
if (processed === files.length) {
668+
// Append to existing content
669+
const current = this.executionMultimedia[type + 's'] || '';
670+
const newContent = current ? current + '\n' + dataURIs.join('\n') : dataURIs.join('\n');
671+
this.executionMultimedia[type + 's'] = newContent;
672+
}
673+
};
674+
675+
reader.readAsDataURL(file);
676+
}
677+
},
555678

556679
async deleteTask(taskId) {
557680
if (!confirm('Are you sure you want to delete this task?')) return;

0 commit comments

Comments
 (0)