Merge branch 'master' into b_whisper_unification

Author: AsyaPronina

samples/CMakeLists.txt

@@ -10,6 +10,7 @@ if(ENABLE_SAMPLES)
     add_subdirectory(cpp/rag)
     add_subdirectory(c/text_generation)
     add_subdirectory(c/whisper_speech_recognition)
+    add_subdirectory(c/visual_language_chat)
 endif()
 
 install(FILES
@@ -41,4 +42,5 @@ install(DIRECTORY
 install(DIRECTORY
         c/text_generation
         c/whisper_speech_recognition
+        c/visual_language_chat
         DESTINATION samples/c COMPONENT cpp_samples_genai)

samples/c/text_generation/chat_sample_c.c

@@ -14,15 +14,15 @@
         fprintf(stderr, "[ERROR] return status %d, line %d\n", return_status, __LINE__); \
         goto err;                                                                        \
     }
-ov_genai_streamming_status_e print_callback(const char* str, void* args) {
+ov_genai_streaming_status_e print_callback(const char* str, void* args) {
     if (str) {
         // If args is not null, it needs to be cast to its actual type.
         fprintf(stdout, "%s", str);
         fflush(stdout);
-        return OV_GENAI_STREAMMING_STATUS_RUNNING;
+        return OV_GENAI_STREAMING_STATUS_RUNNING;
     } else {
         printf("Callback executed with NULL message!\n");
-        return OV_GENAI_STREAMMING_STATUS_STOP;
+        return OV_GENAI_STREAMING_STATUS_STOP;
     }
 }
 

samples/c/visual_language_chat/CMakeLists.txt

@@ -0,0 +1,47 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+find_package(OpenVINOGenAI REQUIRED
+    PATHS
+        "${CMAKE_BINARY_DIR}"  # Reuse the package from the build.
+        ${OpenVINO_DIR}  # GenAI may be installed alogside OpenVINO.
+    NO_CMAKE_FIND_ROOT_PATH
+)
+
+include(FetchContent)
+FetchContent_Declare(
+    stb
+    GIT_REPOSITORY https://github.com/nothings/stb.git
+    GIT_TAG master
+)
+FetchContent_MakeAvailable(stb)
+
+add_library(stb_image INTERFACE)
+target_include_directories(stb_image INTERFACE ${stb_SOURCE_DIR})
+
+# VLM Pipeline Sample
+add_executable(vlm_pipeline_c vlm_pipeline.c load_image.c)
+
+# Specifies that the source file should be compiled as a C source file
+set_source_files_properties(vlm_pipeline.c load_image.c PROPERTIES LANGUAGE C)
+
+target_include_directories(vlm_pipeline_c PRIVATE 
+    ${CMAKE_SOURCE_DIR}/src/c/include
+    ${stb_SOURCE_DIR}
+)
+
+target_link_libraries(vlm_pipeline_c PRIVATE openvino::genai::c stb_image)
+if(UNIX AND NOT APPLE)
+    target_link_libraries(vlm_pipeline_c PRIVATE m)
+endif()
+
+set_target_properties(vlm_pipeline_c PROPERTIES
+    # Ensure out-of-box LC_RPATH on macOS with SIP
+    INSTALL_RPATH_USE_LINK_PATH ON)
+
+# Install
+install(TARGETS vlm_pipeline_c
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)
+

samples/c/visual_language_chat/load_image.c

@@ -0,0 +1,179 @@
+#include "load_image.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _WIN32
+  #define strcasecmp _stricmp
+#else
+  #include <strings.h>
+#endif
+
+#ifdef _WIN32
+    #include <io.h>
+    #define stat _stat
+#else
+    #include <sys/stat.h>
+#endif
+
+#include "openvino/c/openvino.h"
+
+#define STB_IMAGE_IMPLEMENTATION
+#include "stb_image.h"
+
+static const char* supported_extensions[] = {
+    ".jpg", ".jpeg", ".png", ".bmp", ".tga", ".psd", ".gif", ".hdr", ".pic", ".pnm"
+};
+static const size_t num_extensions = sizeof(supported_extensions) / sizeof(supported_extensions[0]);
+
+static int is_supported_image(const char* filename) {
+    if (!filename) return 0;
+    
+    size_t len = strlen(filename);
+    for (size_t i = 0; i < num_extensions; i++) {
+        size_t ext_len = strlen(supported_extensions[i]);
+        if (len >= ext_len) {
+            const char* ext = filename + len - ext_len;
+            if (strcasecmp(ext, supported_extensions[i]) == 0) {
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+typedef struct {
+    unsigned char* image_data;
+    int channels;
+    int height;
+    int width;
+} image_allocator_t;
+
+static void* image_allocate(size_t bytes, size_t alignment, void* user_data) {
+    image_allocator_t* allocator = (image_allocator_t*)user_data;
+    if (allocator && allocator->image_data && 
+        allocator->channels * allocator->height * allocator->width == (int)bytes) {
+        return allocator->image_data;
+    }
+    return NULL;
+}
+
+static void image_deallocate(void* ptr, size_t bytes, size_t alignment, void* user_data) {
+    image_allocator_t* allocator = (image_allocator_t*)user_data;
+    if (allocator && allocator->image_data && 
+        allocator->channels * allocator->height * allocator->width == (int)bytes) {
+        stbi_image_free(allocator->image_data);
+        allocator->image_data = NULL;
+    }
+}
+
+#define CHECK_STATUS(return_status)                                                      \
+    if (return_status != OK) {                                                           \
+        fprintf(stderr, "[ERROR] return status %d, line %d\n", return_status, __LINE__); \
+        goto err;                                                                        \
+    }
+
+ov_tensor_t* load_image(const char* image_path) {
+    if (!image_path) {
+        fprintf(stderr, "Error: image_path is NULL\n");
+        return NULL;
+    }
+    
+    if (!file_exists(image_path)) {
+        fprintf(stderr, "Error: Image file '%s' does not exist\n", image_path);
+        return NULL;
+    }
+    
+    int width, height, channels;
+    const int desired_channels = 3;
+    
+    unsigned char* data = stbi_load(image_path, &width, &height, &channels, desired_channels);
+    if (!data) {
+        fprintf(stderr, "Error: Failed to load image '%s': %s\n", image_path, stbi_failure_reason());
+        return NULL;
+    }
+    
+    image_allocator_t* allocator = (image_allocator_t*)malloc(sizeof(image_allocator_t));
+    if (!allocator) {
+        fprintf(stderr, "Error: Failed to allocate memory for allocator\n");
+        stbi_image_free(data);
+        return NULL;
+    }
+    
+    allocator->image_data = data;
+    allocator->channels = desired_channels;
+    allocator->height = height;
+    allocator->width = width;
+    
+    ov_tensor_t* tensor = NULL;
+    ov_element_type_e input_type = U8;
+    int64_t dims[4] = {1, height, width, desired_channels};
+
+    ov_shape_t input_shape = {.rank = 0, .dims = NULL};
+    ov_shape_create(4, dims, &input_shape);
+
+    ov_tensor_create_from_host_ptr(
+        input_type,
+        input_shape,  // shape: [1, H, W, C]
+        data,
+        &tensor
+    );
+    
+    free(allocator);
+    
+    return tensor;
+}
+
+const ov_tensor_t** load_images(const char* image_path, size_t* tensor_count) {
+    if (!image_path || !tensor_count) {
+        fprintf(stderr, "Error: image_path or tensor_count is NULL\n");
+        return NULL;
+    }
+    
+    if (!file_exists(image_path)) {
+        fprintf(stderr, "Error: Image file '%s' does not exist\n", image_path);
+        return NULL;
+    }
+    
+    ov_tensor_t* tensor = load_image(image_path);
+    if (!tensor) {
+        return NULL;
+    }
+    
+    const ov_tensor_t** tensors = (const ov_tensor_t**)malloc(sizeof(ov_tensor_t*));
+    if (!tensors) {
+        fprintf(stderr, "Error: Failed to allocate memory for single tensor\n");
+        free_tensor(tensor);
+        return NULL;
+    }
+    
+    tensors[0] = tensor;
+    *tensor_count = 1;
+    
+    return tensors;
+}
+
+void free_tensor(ov_tensor_t* tensor) {
+    if (tensor) {
+        ov_tensor_free(tensor);
+    }
+}
+
+void free_tensor_array(ov_tensor_t** tensors, size_t count) {
+    if (tensors) {
+        for (size_t i = 0; i < count; i++) {
+            if (tensors[i]) {
+                ov_tensor_free(tensors[i]);
+            }
+        }
+        free(tensors);
+    }
+}
+
+int file_exists(const char* path) {
+    if (!path) return 0;
+    
+    struct stat buffer;
+    return (stat(path, &buffer) == 0);
+}
+

samples/c/visual_language_chat/load_image.h

@@ -0,0 +1,27 @@
+#ifndef LOAD_IMAGE_H
+#define LOAD_IMAGE_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ov_tensor ov_tensor_t;
+
+ov_tensor_t* load_image(const char* image_path);
+
+const ov_tensor_t** load_images(const char* image_path, size_t* tensor_count);
+
+void free_tensor(ov_tensor_t* tensor);
+
+void free_tensor_array(ov_tensor_t** tensors, size_t count);
+
+int file_exists(const char* path);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // LOAD_IMAGE_H

samples/c/visual_language_chat/vlm_pipeline.c

@@ -0,0 +1,98 @@
+// Copyright (C) 2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * @file vlm_pipeline.c
+ * @brief Example demonstrating how to use OpenVINO GenAI VLM Pipeline C API
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "openvino/genai/c/vlm_pipeline.h"
+#include "load_image.h"
+
+#define MAX_PROMPT_LENGTH 64
+
+// Callback function for streaming results
+ov_genai_streaming_status_e stream_callback(const char* str, void* args) {
+    printf("%s", str);
+    fflush(stdout);
+    return OV_GENAI_STREAMING_STATUS_RUNNING;
+}
+
+int main(int argc, char* argv[]) {
+    if (argc < 4) {
+        printf("Usage: %s <models_path> <device> <image_path> \n", argv[0]);
+        printf("Example: %s ./models CPU ./image.jpg \n", argv[0]);
+        return -1;
+    }
+
+    const char* models_path = argv[1];
+    const char* device = argv[2];
+    const char* image_path = argv[3];
+
+    size_t tensor_count;
+    const ov_tensor_t** tensors = load_images(image_path, &tensor_count);
+
+    // Create VLM pipeline
+    ov_genai_vlm_pipeline* pipeline = NULL;
+    ov_genai_vlm_pipeline_create(models_path, device, 0, &pipeline);
+
+    // Set up streaming callback
+    streamer_callback callback = {
+        .callback_func = stream_callback,
+        .args = NULL
+    };
+
+    // Generate response
+    ov_genai_vlm_decoded_results* results = NULL;
+    ov_genai_generation_config* config = NULL;
+    ov_genai_generation_config_create(&config);
+    ov_genai_generation_config_set_max_new_tokens(config, 100);
+    char prompt[MAX_PROMPT_LENGTH];
+
+    ov_genai_vlm_pipeline_start_chat(pipeline);
+    printf("question:\n");
+
+    if (fgets(prompt, MAX_PROMPT_LENGTH, stdin)) {
+        prompt[strcspn(prompt, "\n")] = 0;
+        if (strlen(prompt) > 0) {
+            ov_genai_vlm_pipeline_generate(pipeline, prompt, tensors, tensor_count, config, &callback, &results);
+            printf("\n----------\nquestion:\n");
+        }
+    }
+
+    while (fgets(prompt, MAX_PROMPT_LENGTH, stdin)) {
+        prompt[strcspn(prompt, "\n")] = 0;
+        if (strlen(prompt) == 0) {
+            continue; 
+        }
+        ov_genai_vlm_pipeline_generate(pipeline, prompt, NULL, 0, config, &callback, &results);
+        printf("\n----------\nquestion:\n");
+    }
+    ov_genai_vlm_pipeline_finish_chat(pipeline);
+
+
+    // Get performance metrics
+    ov_genai_perf_metrics* metrics = NULL;
+    ov_genai_vlm_decoded_results_get_perf_metrics(results, &metrics);
+
+    // Get final result string
+    size_t output_size = 0;
+    ov_genai_vlm_decoded_results_get_string(results, NULL, &output_size);
+    if (output_size > 0) {
+        char* output = (char*)malloc(output_size);
+        if (output) {
+            ov_genai_vlm_decoded_results_get_string(results, output, &output_size);
+            free(output);
+        }
+    }
+
+    // Cleanup
+    ov_genai_vlm_decoded_results_free(results);
+    ov_genai_generation_config_free(config);
+    ov_genai_vlm_pipeline_free(pipeline);
+
+    return 0;
+}