multimodal-rag-nodejs/vertex.js at master · EmpatDevelopment/multimodal-rag-nodejs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
const { GoogleAuth } = require('google-auth-library');

const location = process.env.GCP_LOCATION || 'us-central1';
const dimension = 1408;

const auth = new GoogleAuth({
    scopes: 'https://www.googleapis.com/auth/cloud-platform',
});

async function getClient() {
    const client = await auth.getClient();
    const projectId = await auth.getProjectId();
    const url = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/multimodalembedding@001:predict`;

    return [client, url];
}

async function embedText(text) {
    [client, url] = await getClient();

    const res = await client.request({
        method: 'POST',
        url,
        body: JSON.stringify({
            instances: [
                {
                    text
                }
            ],
            parameters: {
                dimension
            }
        })
    });

    return res.data.predictions[0].textEmbedding;
}

async function embedImage(image) {
    [client, url] = await getClient();

    const res = await client.request({
        method: 'POST',
        url,
        body: JSON.stringify({
            instances: [
                {
                    image: {
                        bytesBase64Encoded: image.buffer.toString('base64')
                    }
                }
            ],
            parameters: {
                dimension
            }
        })
    });

    return res.data.predictions[0].imageEmbedding;
}

async function embedVideo(video) {
    [client, url] = await getClient();

    const res = await client.request({
        method: 'POST',
        url,
        body: JSON.stringify({
            instances: [
                {
                    video: {
                        bytesBase64Encoded: video.buffer.toString('base64'),
                        videoSegmentConfig: {
                            intervalSec: 15
                        }
                    }
                }
            ],
            parameters: {
                dimension
            }
        })
    });

    return res.data.predictions[0].videoEmbeddings;
}

module.exports = {
    embedText,
    embedImage,
    embedVideo
};