Merge branch 'opensearch-project:main' into main

Author: owaiskazi19

.github/workflows/maven-publish.yml

@@ -33,8 +33,14 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: publish snapshots to maven
         run: |

.github/workflows/require-approval.yml

@@ -35,7 +35,15 @@ jobs:
             else
               # Check if the author is in the approvers list
               approvers=$(cat .github/CODEOWNERS | grep @ | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')
-              if [[ "$approvers" =~ "$author" ]]; then
+              is_approved=false
+              IFS=',' read -ra APPROVER_ARRAY <<< "$approvers"
+              for approver in "${APPROVER_ARRAY[@]}"; do
+                if [[ "$approver" == "$author" ]]; then
+                  is_approved=true
+                  break
+                fi
+              done
+              if [[ "$is_approved" == "true" ]]; then
                 echo "$author is in the approval list."
                 echo "is-require-approval=ml-commons-cicd-env" >> $GITHUB_OUTPUT
               else

build-tools/repositories.gradle

@@ -5,8 +5,7 @@
 
 repositories {
     mavenLocal()
-    maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
-    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
+    maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
     mavenCentral()
     maven {url 'https://oss.sonatype.org/content/repositories/snapshots/'}
     maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }

build.gradle

@@ -11,7 +11,7 @@ buildscript {
     ext {
         opensearch_group = "org.opensearch"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        opensearch_version = System.getProperty("opensearch.version", "3.3.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.4.0-SNAPSHOT")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         asm_version = "9.7"
 
@@ -31,8 +31,7 @@ buildscript {
 
     repositories {
         mavenLocal()
-        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
-        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
         mavenCentral()
         maven { url "https://plugins.gradle.org/m2/" }
         maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }

client/build.gradle

@@ -72,10 +72,11 @@ publishing {
         }
         maven {
             name = "Snapshots"
-            url = "https://central.sonatype.com/repository/maven-snapshots/"
-            credentials {
-                username "$System.env.SONATYPE_USERNAME"
-                password "$System.env.SONATYPE_PASSWORD"
+            url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO")
+            credentials(AwsCredentials) {
+                accessKey = System.getenv("AWS_ACCESS_KEY_ID")
+                secretKey = System.getenv("AWS_SECRET_ACCESS_KEY")
+                sessionToken = System.getenv("AWS_SESSION_TOKEN")
             }
         }
     }

common/build.gradle

@@ -22,7 +22,9 @@ dependencies {
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.15.2'
     testImplementation "org.opensearch.test:framework:${opensearch_version}"
 
-    compileOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
+    compileOnly group: 'org.opensearch', name:'opensearch-security-spi', version:"${opensearch_build}"
+
+    compileOnly group: 'org.apache.commons', name: 'commons-text', version: '1.14.0'
     compileOnly group: 'com.google.code.gson', name: 'gson', version: "${versions.gson}"
     compileOnly group: 'org.json', name: 'json', version: '20231013'
     testImplementation group: 'org.json', name: 'json', version: '20231013'
@@ -111,10 +113,11 @@ publishing {
         }
         maven {
             name = "Snapshots" //  optional target repository name
-            url = "https://central.sonatype.com/repository/maven-snapshots/"
-            credentials {
-                username "$System.env.SONATYPE_USERNAME"
-                password "$System.env.SONATYPE_PASSWORD"
+            url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO")
+            credentials(AwsCredentials) {
+                accessKey = System.getenv("AWS_ACCESS_KEY_ID")
+                secretKey = System.getenv("AWS_SECRET_ACCESS_KEY")
+                sessionToken = System.getenv("AWS_SESSION_TOKEN")
             }
         }
     }

common/src/main/java/org/opensearch/ml/common/CommonValue.java

@@ -46,7 +46,9 @@ public class CommonValue {
     public static final String ML_AGENT_INDEX = ".plugins-ml-agent";
     public static final String ML_MEMORY_META_INDEX = ".plugins-ml-memory-meta";
     public static final String ML_MEMORY_MESSAGE_INDEX = ".plugins-ml-memory-message";
-    public static final String ML_MEMORY_CONTAINER_INDEX = ".plugins-ml-memory-container";
+    public static final String ML_MEMORY_CONTAINER_INDEX = ".plugins-ml-am-memory-container";
+    public static final String ML_AGENTIC_MEMORY_SYSTEM_INDEX_PREFIX = ".plugins-ml-am";
+    public static final String ML_AGENTIC_MEMORY_INDEX_PATTERN = ML_AGENTIC_MEMORY_SYSTEM_INDEX_PREFIX + "*";
     public static final String ML_STOP_WORDS_INDEX = ".plugins-ml-stop-words";
     // index used in 2.19 to track MlTaskBatchUpdate task
     public static final String TASK_POLLING_JOB_INDEX = ".ml_commons_task_polling_job";
@@ -68,12 +70,19 @@ public class CommonValue {
     public static final String ML_MEMORY_META_INDEX_MAPPING_PATH = "index-mappings/ml_memory_meta.json";
     public static final String ML_MEMORY_MESSAGE_INDEX_MAPPING_PATH = "index-mappings/ml_memory_message.json";
     public static final String ML_MEMORY_CONTAINER_INDEX_MAPPING_PATH = "index-mappings/ml_memory_container.json";
+    public static final String ML_MEMORY_SESSION_INDEX_MAPPING_PATH = "index-mappings/ml_memory_sessions.json";
+    public static final String ML_WORKING_MEMORY_INDEX_MAPPING_PATH = "index-mappings/ml_memory_working.json";
+    public static final String ML_LONG_TERM_MEMORY_INDEX_MAPPING_PATH = "index-mappings/ml_memory_long_term.json";
+    public static final String ML_LONG_MEMORY_HISTORY_INDEX_MAPPING_PATH = "index-mappings/ml_memory_long_term_history.json";
     public static final String ML_MCP_SESSION_MANAGEMENT_INDEX_MAPPING_PATH = "index-mappings/ml_mcp_session_management.json";
     public static final String ML_MCP_TOOLS_INDEX_MAPPING_PATH = "index-mappings/ml_mcp_tools.json";
     public static final String ML_JOBS_INDEX_MAPPING_PATH = "index-mappings/ml_jobs.json";
     public static final String ML_INDEX_INSIGHT_CONFIG_INDEX_MAPPING_PATH = "index-mappings/ml_index_insight_config.json";
     public static final String ML_INDEX_INSIGHT_STORAGE_INDEX_MAPPING_PATH = "index-mappings/ml_index_insight_storage.json";
 
+    // Resource type used in resource-access-control
+    public static final String ML_MODEL_GROUP_RESOURCE_TYPE = "ml-model-group";
+
     // Calculate Versions independently of OpenSearch core version
     public static final Version VERSION_2_11_0 = Version.fromString("2.11.0");
     public static final Version VERSION_2_12_0 = Version.fromString("2.12.0");

common/src/main/java/org/opensearch/ml/common/MLModel.java

@@ -863,17 +863,17 @@ public Tags getTags() {
     public Tags getTags(Connector connector) {
         // if connector is present, model is a remote model
         if (this.algorithm == FunctionName.REMOTE && connector != null) {
-            return getRemoteModelTags(connector);
+            return getRemoteModelTags(connector).addTag(IS_HIDDEN_FIELD, isHidden != null && isHidden);
         }
 
         // pre-trained models follow a specific naming convention, relying on that to identify a pre-trained model
         if (this.name != null
             && (this.name.startsWith("amazon/") || this.name.startsWith("huggingface/"))
             && this.name.split("/").length >= 3) {
-            return getPreTrainedModelTags();
+            return getPreTrainedModelTags().addTag(IS_HIDDEN_FIELD, isHidden != null && isHidden);
         }
 
-        return getCustomModelTags();
+        return getCustomModelTags().addTag(IS_HIDDEN_FIELD, isHidden != null && isHidden);
     }
 
     /**

common/src/main/java/org/opensearch/ml/common/MLModelGroup.java

@@ -6,6 +6,7 @@
 package org.opensearch.ml.common;
 
 import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.opensearch.ml.common.CommonValue.BACKEND_ROLES_FIELD;
 import static org.opensearch.ml.common.CommonValue.TENANT_ID_FIELD;
 import static org.opensearch.ml.common.CommonValue.VERSION_2_19_0;
 
@@ -33,7 +34,6 @@ public class MLModelGroup implements ToXContentObject {
     public static final String MODEL_GROUP_NAME_FIELD = "name"; // name of the model group
     public static final String DESCRIPTION_FIELD = "description"; // description of the model group
     public static final String LATEST_VERSION_FIELD = "latest_version"; // latest model version added to the model group
-    public static final String BACKEND_ROLES_FIELD = "backend_roles"; // back_end roles as specified by the owner/admin
     public static final String OWNER = "owner"; // user who creates/owns the model group
 
     public static final String ACCESS = "access"; // assigned to public, private, or null when model group created

common/src/main/java/org/opensearch/ml/common/ResourceSharingClientAccessor.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.ml.common;
+
+import org.opensearch.security.spi.resources.client.ResourceSharingClient;
+
+/**
+ * Accessor for resource sharing client
+ */
+public class ResourceSharingClientAccessor {
+    private ResourceSharingClient CLIENT;
+
+    private static ResourceSharingClientAccessor resourceSharingClientAccessor;
+
+    private ResourceSharingClientAccessor() {}
+
+    public static ResourceSharingClientAccessor getInstance() {
+        if (resourceSharingClientAccessor == null) {
+            resourceSharingClientAccessor = new ResourceSharingClientAccessor();
+        }
+
+        return resourceSharingClientAccessor;
+    }
+
+    /**
+     * Set the resource sharing client
+     */
+    public void setResourceSharingClient(ResourceSharingClient client) {
+        resourceSharingClientAccessor.CLIENT = client;
+    }
+
+    /**
+     * Get the resource sharing client
+     */
+    public ResourceSharingClient getResourceSharingClient() {
+        return resourceSharingClientAccessor.CLIENT;
+    }
+
+}