openhab · jimtng · May 11, 2025 · May 22, 2025 · May 26, 2025 · May 26, 2025
@@ -27,7 +27,7 @@
 
 import org.eclipse.jdt.annotation.NonNullByDefault;
 import org.openhab.core.automation.module.script.ScriptDependencyTracker;
-import org.openhab.core.automation.module.script.rulesupport.internal.loader.BidiSetBag;
+import org.openhab.core.common.BidiSetBag;
 import org.openhab.core.service.WatchService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

@@ -13,7 +13,6 @@
 package org.openhab.core.model.yaml.internal;
 
 import static org.openhab.core.model.yaml.YamlModelUtils.*;
-import static org.openhab.core.service.WatchService.Kind.CREATE;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -38,13 +37,15 @@
 
 import org.eclipse.jdt.annotation.NonNullByDefault;
 import org.eclipse.jdt.annotation.Nullable;
+import org.openhab.core.common.BidiSetBag;
 import org.openhab.core.model.yaml.YamlElement;
 import org.openhab.core.model.yaml.YamlElementName;
 import org.openhab.core.model.yaml.YamlModelListener;
 import org.openhab.core.model.yaml.YamlModelRepository;
 import org.openhab.core.model.yaml.internal.items.YamlItemDTO;
 import org.openhab.core.model.yaml.internal.semantics.YamlSemanticTagDTO;
 import org.openhab.core.model.yaml.internal.things.YamlThingDTO;
+import org.openhab.core.model.yaml.internal.util.preprocessor.YamlPreprocessor;
 import org.openhab.core.service.WatchService;
 import org.openhab.core.service.WatchService.Kind;
 import org.osgi.service.component.annotations.Activate;
@@ -81,6 +82,7 @@
  * @author Laurent Garnier - new parameters to retrieve errors and warnings when loading a file
  * @author Laurent Garnier - Added methods addElementsToBeGenerated, generateFileFormat, createIsolatedModel and
  *         removeIsolatedModel
+ * @author Jimmy Tanagra - Added Yaml preprocessor to support !include, variable substitutions, and packages
  */
 @NonNullByDefault
 @Component(immediate = true)
@@ -89,6 +91,8 @@ public class YamlModelRepositoryImpl implements WatchService.WatchEventListener,
     private static final String VERSION = "version";
     private static final String READ_ONLY = "readOnly";
     private static final Set<String> KNOWN_ELEMENTS = Set.of( //
+            // "version", "readOnly" are reserved keys
+            // "variables" and "packages" are reserved elements for YamlPreprocessor
             getElementName(YamlSemanticTagDTO.class), // "tags"
             getElementName(YamlThingDTO.class), // "things"
             getElementName(YamlItemDTO.class) // "items"
@@ -113,15 +117,20 @@ public class YamlModelRepositoryImpl implements WatchService.WatchEventListener,
 
     private int counter;
 
+    // keep track of include files so we can reload the main model when they change
+    // Bidirectional Map of modelName <-> include path by this model
-    // keep track of include files so we can reload the main model when they change
-    // Bidirectional Map of modelName <-> include path by this model
+    /**
+     * A bidirectional mapping between model names and their include file paths.
+     * 
+     * <p>This field is used to track dependencies between YAML models and their include files.
+     * Each model name (represented as a string) is mapped to the paths of the files it includes,
+     * and vice versa. This bidirectional mapping allows efficient tracking and reloading of models
+     * when their include files are modified.</p>
+     * 
+     * <p>For example, if a YAML model includes another file, the mapping ensures that changes to
+     * the included file trigger a reload of the main model. This is critical for maintaining
+     * consistency and ensuring that the system reflects the latest configuration.</p>
+     */
-    // keep track of include files so we can reload the main model when they change
-    // Bidirectional Map of modelName <-> include path by this model
+    /**
+     * A bidirectional mapping between model names and their include file paths.
+     * 
+     * <p>This field is used to track dependencies between YAML models and their include files.
+     * Each model name (represented as a string) is mapped to the paths of the files it includes,
+     * and vice versa. This bidirectional mapping allows efficient tracking and reloading of models
+     * when their include files are modified.</p>
+     * 
+     * <p>For example, if a YAML model includes another file, the mapping ensures that changes to
+     * the included file trigger a reload of the main model. This is critical for maintaining
+     * consistency and ensuring that the system reflects the latest configuration.</p>
+     */
+    private final BidiSetBag<String, Path> modelIncludes = new BidiSetBag<>();
+    private boolean initializing = true;
+
     @Activate
     public YamlModelRepositoryImpl(@Reference(target = WatchService.CONFIG_WATCHER_FILTER) WatchService watchService) {
         YAMLFactory yamlFactory = YAMLFactory.builder() //
                 .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) // omit "---" at file start
                 .disable(YAMLGenerator.Feature.SPLIT_LINES) // do not split long lines
                 .enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR) // indent arrays
                 .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) // use quotes only where necessary
-                .enable(YAMLParser.Feature.PARSE_BOOLEAN_LIKE_WORDS_AS_STRINGS).build(); // do not parse ON/OFF/... as
-                                                                                         // booleans
+                .enable(YAMLParser.Feature.PARSE_BOOLEAN_LIKE_WORDS_AS_STRINGS) // do not parse ON/OFF/... as booleans
+                .build();
         this.objectMapper = new ObjectMapper(yamlFactory);
         objectMapper.findAndRegisterModules();
         objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
@@ -149,7 +158,7 @@ public YamlModelRepositoryImpl(@Reference(target = WatchService.CONFIG_WATCHER_F
                     public FileVisitResult visitFile(@NonNullByDefault({}) Path file,
                             @NonNullByDefault({}) BasicFileAttributes attrs) throws IOException {
                         if (attrs.isRegularFile()) {
-                            processWatchEvent(CREATE, file);
+                            processWatchEvent(Kind.CREATE, file);
                         }
                         return FileVisitResult.CONTINUE;
                     }
@@ -173,30 +182,46 @@ public FileVisitResult visitFileFailed(@NonNullByDefault({}) Path file,
                         e.getMessage());
             }
         });
+        initializing = false;
     }
 
     @Deactivate
     public void deactivate() {
         watchService.unregisterListener(this);
+        modelIncludes.clear();
     }
 
     // The method is "synchronized" to avoid concurrent files processing
     @Override
     public synchronized void processWatchEvent(Kind kind, Path fullPath) {
         Path relativePath = mainWatchPath.relativize(fullPath);
         String modelName = relativePath.toString();
-        if (!modelName.endsWith(".yaml") && !modelName.endsWith(".yml")) {
+
+        // always clear the list of includes if it's a model
+        // if it loads correctly, it will be re-populated
+        modelIncludes.removeKey(modelName);
+
+        // check here because include files can have any extension
+        if (!initializing && processIncludeFile(kind, fullPath)) {
+            return;
+        }
+
+        if ((!modelName.endsWith(".yaml") && !modelName.endsWith(".yml")) || modelName.endsWith(".inc.yaml")
+                || modelName.endsWith(".inc.yml")) {
             logger.trace("Ignored {}", fullPath);
             return;
         }
 
         List<String> errors = new ArrayList<>();
         List<String> warnings = new ArrayList<>();
         try {
-            if (kind == WatchService.Kind.DELETE) {
+            if (kind == Kind.DELETE) {
                 removeModel(modelName);
             } else if (!Files.isHidden(fullPath) && Files.isReadable(fullPath) && !Files.isDirectory(fullPath)) {
-                processModelContent(modelName, kind, objectMapper.readTree(fullPath.toFile()), errors, warnings);
+                Object yamlObject = YamlPreprocessor.load(fullPath, includePath -> {
+                    modelIncludes.put(modelName, includePath);
+                });
+                processModelContent(modelName, kind, objectMapper.valueToTree(yamlObject), errors, warnings);
             } else {
                 logger.trace("Ignored {}", fullPath);
             }
@@ -331,13 +356,45 @@ private boolean processModelContent(String modelName, Kind kind, JsonNode fileCo
         return valid;
     }
 
+    private boolean processIncludeFile(Kind kind, Path fullPath) {
+        boolean logged = false;
+
+        Set<String> dependingModels = modelIncludes.getKeys(fullPath);
+
+        if (dependingModels.isEmpty()) {
+            return false;
+        }
+
+        String action = switch (kind) {
+            case CREATE -> "created";
+            case DELETE -> "deleted";
+            case MODIFY -> "modified";
+            default -> kind.name();
+        };
+        logger.info("An include file '{}' was {}", fullPath, action);
+
+        dependingModels.forEach(modelName -> {
+            Path modelPath = mainWatchPath.resolve(modelName);
+            try {
+                // reprocess the model that depends on this include file
+                processWatchEvent(Kind.MODIFY, modelPath);
+            } catch (Exception e) {
+                logger.warn("Failed to reprocess model {} after include file change: {}", modelName, e.getMessage());
+            }
+        });
+
+        return true;
+    }
+
     @SuppressWarnings({ "rawtypes", "unchecked" })
     private void removeModel(String modelName) {
         YamlModelWrapper removedModel = modelCache.remove(modelName);
         if (removedModel == null) {
             return;
         }
         logger.info("Removing YAML model {}", modelName);
+        modelIncludes.removeKey(modelName);
+
         int version = removedModel.getVersion();
         for (Map.Entry<String, @Nullable JsonNode> modelEntry : removedModel.getNodes().entrySet()) {
             String elementName = modelEntry.getKey();

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2010-2025 Contributors to the openHAB project
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.openhab.core.model.yaml.internal.util.preprocessor;
+
+import java.util.Map;
+
+/**
+ * The {@link IncludeObject} represents an object constructed from an <code>!include</code> node
+ * to be processed by the {@link YamlPreprocessor}.
+ *
+ * @author Jimmy Tanagra - Initial contribution
+ */
+record IncludeObject(String fileName, Map<String, String> vars) {
+}