Update Terraform documentation (1.15.8)

Fixes #2590.

Author: simon04

lib/docs/filters/terraform/clean_html.rb

@@ -1,33 +1,62 @@
 module Docs
   class Terraform
     class CleanHtmlFilter < Filter
+      # Prism doesn't ship every language the docs use; map those onto the
+      # closest grammar it does know about.
+      LANGUAGE_ALIASES = {
+        'console' => 'shellsession',
+        # HCL isn't currently supported by Prism, Ruby syntax does an acceptable job for now
+        'hcl'     => 'ruby',
+        'tf'      => 'ruby',
+        'terraform' => 'ruby',
+      }
+
+      # Languages that carry no syntax at all.
+      PLAIN_LANGUAGES = %w(documentation plaintext text txt none)
+
       def call
-        @doc = at_css('#inner')
+        @doc = at_css('#main')
+
+        # Chrome that isn't part of the documentation itself.
+        css('[class*="versionSwitcher"]', '[class*="copy-button"]', 'svg', 'hr').remove
+        css('a[aria-label$=" permalink"]', 'a[href*="web-unified-docs/blob"]').remove
 
-        css('hr', 'a.anchor').remove
+        # Lift the page title out of its wrapper divs.
+        if heading = at_css('[class*="docs-page-heading_root"]')
+          heading.css('h1').each { |node| node.remove_attribute('class') }
+          heading.before(heading.css('h1')).remove
+        end
 
         css('.alert').each do |node|
           node.name = 'blockquote'
         end
 
         css('pre').each do |node|
-          if language = node['class'][/(json|shell|ruby)/, 1]
+          if language = language_for(node)
             node['data-language'] = language
           end
-          # HCL isn't currently supported by Prism, Ruby syntax does an acceptable job for now
-          if language = node['class'][/(hcl)/, 1]
-            node['data-language'] = 'ruby'
-          end
+          # Drop the syntax highlighting markup so that Prism can redo it.
           node.content = node.content
           node.remove_attribute('class')
-        end
-
-        css('.highlight').each do |node|
-          node.before(node.children).remove
+          node.remove_attribute('id')
+          node.remove_attribute('style')
         end
 
         doc
       end
+
+      private
+
+      # The language is declared on a wrapper around the <pre>, as "language-hcl".
+      def language_for(node)
+        while (node = node.parent) && node.element?
+          next unless language = node['class'].to_s[/(?:\A|\s)language-([\w-]+)/, 1]
+          language = language.downcase
+          return nil if PLAIN_LANGUAGES.include?(language)
+          return LANGUAGE_ALIASES[language] || language
+        end
+        nil
+      end
     end
   end
 end

lib/docs/filters/terraform/entries.rb

@@ -1,75 +1,61 @@
 module Docs
   class Terraform
     class EntriesFilter < Docs::EntriesFilter
-
-      # Some providers have non-trivial mappings between the directory they live in and their name
-      # Anything *not* in this list will be capitalized instead.
-      PROVIDER_NAME_MAP = {
-        'aws'              => 'AWS',
-        'azure'            => 'Azure (Legacy)',
-        'azurerm'          => 'Azure',
-        'centurylinkcloud' => 'CenturyLinkCloud',
-        'cloudscale'       => 'CloudScale.ch',
-        'cloudstack'       => 'CloudStack',
-        'dme'              => 'DNSMadeEasy',
-        'dns'              => 'DNS',
-        'dnsimple'         => 'DNSimple',
-        'do'               => 'DigitalOcean',
-        'github'           => 'GitHub',
-        'google'           => 'Google Cloud',
-        'http'             => 'HTTP',
-        'mysql'            => 'MySQL',
-        'newrelic'         => 'New Relic',
-        'oneandone'        => '1&1',
-        'opentelekomcloud' => 'OpenTelekomCloud',
-        'opsgenie'         => 'OpsGenie',
-        'opc'              => 'Oracle Public Cloud',
-        'oraclepaas'       => 'Oracle Cloud Platform',
-        'ovh'              => 'OVH',
-        'pagerduty'        => 'PagerDuty',
-        'panos'            => 'Palo Alto Networks',
-        'postgresql'       => 'PostgreSQL',
-        'powerdns'         => 'PowerDNS',
-        'profitbricks'     => 'ProfitBricks',
-        'rabbitmq'         => 'RabbitMQ',
-        'softlayer'        => 'SoftLayer',
-        'statuscake'       => 'StatusCake',
-        'tls'              => 'TLS',
-        'ultradns'         => 'UltraDNS',
-        'vcd'              => 'VMware vCloud Director',
-        'nsxt'             => 'VMware NSX-T',
-        'vsphere'          => 'VMware vSphere',
-      }
-
-      # Some providers have a lot (> 100) entries, which makes browsing them unwieldy.
-      # Any present in the list below will have an extra set of types added, breaking the pages out into the different
-      # products they offer.
-      LARGE_PROVIDERS = {
-        "aws"     => true,
-        "azurerm" => true,
-        "google"  => true,
+      # Fallback types for pages whose sidebar navigation cannot be interpreted,
+      # keyed by the first segment of the slug.
+      SECTION_NAME_MAP = {
+        'cdktf'      => 'CDK for Terraform',
+        'cli'        => 'Terraform CLI',
+        'docs'       => 'Documentation',
+        'intro'      => 'Intro to Terraform',
+        'internals'  => 'Internals',
+        'language'   => 'Configuration Language',
+        'mcp-server' => 'MCP Server',
+        'migrate'    => 'Migrate',
+        'plugin'     => 'Plugin Development',
+        'policy'     => 'Policy Enforcement',
+        'registry'   => 'Registry Publishing',
+        'tutorials'  => 'Tutorials',
       }
 
-
       def get_name
-        name ||= at_css('#inner h1').content
-        name.remove! "» "
-        name.remove! "Data Source: "
-        name
+        at_css('#main h1').content.strip
       end
 
       def get_type
-        category, subcategory, subfolder, page = *slug.split('/')
-        provider = page ? subcategory : category
-        nice_provider = PROVIDER_NAME_MAP[provider] || provider.capitalize
+        # The sidebar heading names the section the page belongs to, e.g. "Terraform CLI".
+        section = at_css('#sidebar-label').try(:content).try(:strip)
+        root = slug.split('/').first
+        section = SECTION_NAME_MAP[root] if section.blank?
+        return 'Terraform' if section.blank?
+
+        # Tutorials reuse the same section names as the reference docs
+        # (e.g. "CLI"), so keep the two apart.
+        return "Tutorials: #{section}" if root == 'tutorials' && section != 'Tutorials'
+
+        group = sidebar_group
+        group.present? ? "#{section}: #{group}" : section
+      end
+
+      private
+
+      # Returns the label of the outermost collapsible group containing the
+      # current page in the sidebar, e.g. "Provisioning Infrastructure".
+      def sidebar_group
+        nav = at_css('#sidebar-nav')
+        return if nav.nil?
+
+        node = nav.at_css('a[aria-current="page"]')
+        label = nil
 
-        if LARGE_PROVIDERS[provider]
-          category_node = at_css('ul > li > ul > li.active')
-          parent_node = category_node.parent.previous_element if category_node
-          nice_provider = nice_provider + ": #{parent_node.content}" if category_node
+        while node && node != nav
+          node = node.parent
+          next unless node && node.name == 'li'
+          button = node.at_css('> button > span')
+          label = button.content.strip if button
         end
 
-        nice_provider
+        label
       end
     end
   end

lib/docs/scrapers/terraform.rb

@@ -2,26 +2,34 @@ module Docs
   class Terraform < UrlScraper
     self.name = 'Terraform'
     self.type = 'terraform'
-    self.release = '0.11.7'
-    self.base_url = 'https://www.terraform.io/docs/'
-    self.root_path = 'index.html'
+    self.release = '1.15.8'
+    self.base_url = 'https://developer.hashicorp.com/terraform/'
+    self.root_path = 'docs'
+    self.initial_paths = %w(internals)
     self.links = {
       home: 'https://www.terraform.io/',
       code: 'https://github.com/hashicorp/terraform'
     }
 
     html_filters.push 'terraform/entries', 'terraform/clean_html'
 
-    options[:skip_patterns] = [/enterprise/, /enterprise-legacy/]
+    options[:trailing_slash] = false
+
+    options[:skip_patterns] = [
+      # HCP Terraform / Terraform Enterprise are separate products
+      /\Aenterprise/,
+      /\Acloud-docs/,
+      # The version switcher links to every past release of the same page
+      %r{(\A|/)v\d+\.\d+\.x(/|\z)},
+    ]
 
     options[:attribution] = <<-HTML
-      &copy; 2018 HashiCorp</br>
+      &copy; 2026 HashiCorp<br>
       Licensed under the MPL 2.0 License.
     HTML
 
     def get_latest_version(opts)
-      contents = get_latest_github_release('hashicorp', 'terraform', opts)
-      contents.sub("v", "")
+      get_latest_github_release('hashicorp', 'terraform', opts)
     end
   end
 end