Sync eng/common directory with azure-sdk-tools for PR 1219 (#13710)

* Move entire docgeneration into common tools * Move docindex to common * Added the package replacement logic * Fixed on parameters * Fixed param * Change function to dash * Added regex on function * Added display name. * Update eng/common/docgeneration/Generate-DocIndex.ps1 Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com> * Deal with js * Add no new line args * revert some test changes * Need to default to the double quotes for JS regex * Update Generate-DocIndex.ps1 * Added the appTitle * type Co-authored-by: Sima Zhu <sizhu@microsoft.com> Co-authored-by: Sima Zhu <48036328+sima-zhu@users.noreply.github.com> Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>
Azure · Nov 20, 2020 · 8d7d6ee · 8d7d6ee
1 parent 6b341a6
commit 8d7d6ee
Show file tree

Hide file tree

Showing 11 changed files with 1,317 additions and 0 deletions.
diff --git a/eng/common/docgeneration/Generate-DocIndex.ps1 b/eng/common/docgeneration/Generate-DocIndex.ps1
@@ -0,0 +1,196 @@
+# Generates an index page for cataloging different versions of the Docs
+[CmdletBinding()]
+Param (
+    $DocFx,
+    $RepoRoot,
+    $DocGenDir,
+    $DocOutDir = "${RepoRoot}/docfx_project",
+    $DocfxJsonPath = "${PSScriptRoot}\docfx.json",
+    $MainJsPath = "${PSScriptRoot}\templates\matthews\styles\main.js"
+)
+. "${PSScriptRoot}\..\scripts\common.ps1"
+$GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex"
+
+# Given the metadata url under https://github.com/Azure/azure-sdk/tree/master/_data/releases/latest, 
+# the function will return the csv metadata back as part of response.
+function Get-CSVMetadata ([string]$MetadataUri) {
+    $metadataResponse = Invoke-RestMethod -Uri $MetadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv
+    return $metadataResponse
+}
+
+# Given the github io blob storage url and language regex,
+# the helper function will return a list of artifact names.
+function Get-BlobStorage-Artifacts($blobStorageUrl, $blobDirectoryRegex, $blobArtifactsReplacement) {
+    LogDebug "Reading artifact from storage blob ..."
+    $returnedArtifacts = @()
+    $pageToken = ""
+    Do {
+      $resp = ""
+      if (!$pageToken) {
+        # First page call.
+        $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrl
+      }
+      else {
+        # Next page call
+        $blobStorageUrlPageToken = $blobStorageUrl + "&marker=$pageToken"
+        $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrlPageToken
+      }
+      # Convert to xml documents. 
+      $xmlDoc = [xml](removeBomFromString $resp)
+      foreach ($elem in $xmlDoc.EnumerationResults.Blobs.BlobPrefix) {
+        # What service return like "dotnet/Azure.AI.Anomalydetector/", needs to fetch out "Azure.AI.Anomalydetector"
+        $artifact = $elem.Name -replace $blobDirectoryRegex, $blobArtifactsReplacement
+        $returnedArtifacts += $artifact
+      }
+      # Fetch page token
+      $pageToken = $xmlDoc.EnumerationResults.NextMarker
+    } while ($pageToken)
+    return $returnedArtifacts
+  }
+
+# The sequence of Bom bytes differs by different encoding. 
+# The helper function here is only to strip the utf-8 encoding system as it is used by blob storage list api.
+# Return the original string if not in BOM utf-8 sequence.
+function RemoveBomFromString([string]$bomAwareString) {
+    if ($bomAwareString.length -le 3) {
+        return $bomAwareString
+    }
+    $bomPatternByteArray = [byte[]] (0xef, 0xbb, 0xbf)
+    # The default encoding for powershell is ISO-8859-1, so converting bytes with the encoding.
+    $bomAwareBytes = [Text.Encoding]::GetEncoding(28591).GetBytes($bomAwareString.Substring(0, 3))
+    if (@(Compare-Object $bomPatternByteArray $bomAwareBytes -SyncWindow 0).Length -eq 0) {
+        return $bomAwareString.Substring(3)
+    }
+    return $bomAwareString
+}
+
+function Get-TocMapping { 
+    Param (
+        [Parameter(Mandatory = $true)] [Object[]] $metadata,
+        [Parameter(Mandatory = $true)] [String[]] $artifacts
+    )
+    # Used for sorting the toc display order
+    $orderServiceMapping = @{}
+
+    foreach ($artifact in $artifacts) {
+        $packageInfo = $metadata | ? {$_.Package -eq $artifact}
+
+        if ($packageInfo -and $packageInfo[0].Hide -eq 'true') {
+            LogDebug "The artifact $artifact set 'Hide' to 'true'."
+            continue
+        }
+        $serviceName = ""
+        $displayName = ""
+        if (!$packageInfo) {
+            LogWarning "There is no artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. "
+            # If no service name retrieved, print out warning message, and put it into Other page.
+            $serviceName = "Other"
+        }
+        elseif (!$packageInfo[0].ServiceName) {
+            LogWarning "There is no service name for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. "
+            # If no service name retrieved, print out warning message, and put it into Other page.
+            $serviceName = "Other"
+            $displayName = $packageInfo[0].DisplayName.Trim()
+        }
+        else {
+            if ($packageInfo.Length -gt 1) {
+                LogWarning "There are more than 1 packages fetched out for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. "
+            }
+            $serviceName = $packageInfo[0].ServiceName.Trim()
+            $displayName = $packageInfo[0].DisplayName.Trim()
+        }
+        $orderServiceMapping[$artifact] = @($serviceName, $displayName)
+    }
+    return $orderServiceMapping                   
+}
+
+function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang) {
+    LogDebug "Start generating the docfx toc and build docfx site..."
+
+    LogDebug "Initializing Default DocFx Site..."
+    & $($DocFx) init -q -o "${DocOutDir}"
+    # The line below is used for testing in local
+    #docfx init -q -o "${DocOutDir}"
+    LogDebug "Copying template and configuration..."
+    New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Force
+    Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse
+    Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force
+    $YmlPath = "${DocOutDir}/api"
+    New-Item -Path $YmlPath -Name "toc.yml" -Force
+    $visitedService = @{}
+    # Sort and display toc service name by alphabetical order, and then sort artifact by order.
+    foreach ($serviceMapping in ($tocContent.GetEnumerator() | Sort-Object Value[0], Key)) {
+        $artifact = $serviceMapping.Key
+        $serviceName = $serviceMapping.Value[0]
+        $displayName = $serviceMapping.Value[1]
+
+        $fileName = ($serviceName -replace '\s', '').ToLower().Trim()
+        if ($visitedService.ContainsKey($serviceName)) {
+            if ($displayName) {
+                Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact`n##### ($displayName)"
+            }
+            else {
+                Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact"
+            }
+        }
+        else {
+            Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: ${serviceName}`r`n  href: ${fileName}.md"
+            New-Item -Path $YmlPath -Name "${fileName}.md" -Force
+            if ($displayName) {
+                Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact`n##### ($displayName)"
+            }
+            else {
+                Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact"
+            }
+            $visitedService[$serviceName] = $true
+        }
+    }
+
+    # Generate toc homepage.
+    LogDebug "Creating Site Title and Navigation..."
+    New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force
+    Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for $lang APIs`r`n  href: api/`r`n  homepage: api/index.md"
+
+    LogDebug "Copying root markdowns"
+    Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force
+    Copy-Item "$($RepoRoot)/CONTRIBUTING.md" -Destination "${DocOutDir}/api/CONTRIBUTING.md" -Force
+
+    LogDebug "Building site..."
+    & $($DocFx) build "${DocOutDir}/docfx.json"
+    # The line below is used for testing in local
+    #docfx build "${DocOutDir}/docfx.json"
+    Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force    
+}
+
+function Mutate-Files {
+    Param (
+        [Parameter(Mandatory=$true)] [String]$appTitle,
+        [Parameter(Mandatory=$true)] [String]$lang,
+        [Parameter(Mandatory=$true)] [String]$indexhtmlloc,
+        [Parameter(Mandatory=$false)] [String]$packageRegex = "`"`"",
+        [Parameter(Mandatory=$false)] [String]$regexReplacement = ""
+    )
+    # Update docfx.json
+    $docfxContent = Get-Content -Path $DocfxJsonPath -Raw
+    $docfxContent = $docfxContent -replace "`"_appTitle`": `"`"", "`"_appTitle`": `"$appTitle`""
+    $docfxContent = $docfxContent -replace "`"_appFooter`": `"`"", "`"_appFooter`": `"$appTitle`""
+    Set-Content -Path $DocfxJsonPath -Value $docfxContent
+    # Update main.js var lang
+    $mainJsContent = Get-Content -Path $MainJsPath -Raw
+    $mainJsContent = $mainJsContent -replace "var SELECTED_LANGUAGE = ''", "var SELECTED_LANGUAGE = '$lang'"
+    # Update main.js var index html
+    $mainJsContent = $mainJsContent -replace "var INDEX_HTML = ''", "var INDEX_HTML = '$indexhtmlloc'"
+    # Update main.js package regex and replacement
+    $mainJsContent = $mainJsContent -replace "var PACKAGE_REGEX = ''", "var PACKAGE_REGEX = $packageRegex"
+    $mainJsContent = $mainJsContent -replace "var PACKAGE_REPLACEMENT = ''", "var PACKAGE_REPLACEMENT = `"$regexReplacement`""
+    Set-Content -Path $MainJsPath -Value $mainJsContent -NoNewline
+}
+
+if ($GetGithubIoDocIndexFn -and (Test-Path "function:$GetGithubIoDocIndexFn"))
+{
+    &$GetGithubIoDocIndexFn
+}
+else
+{
+    LogWarning "The function '$GetGithubIoDocIndexFn' was not found."
+}
diff --git a/eng/common/docgeneration/assets/logo.svg b/eng/common/docgeneration/assets/logo.svg
diff --git a/eng/common/docgeneration/docfx.json b/eng/common/docgeneration/docfx.json
@@ -0,0 +1,78 @@
+{
+  "metadata": [
+    {
+      "src": [
+        {
+          "files": [
+            "src/**.csproj"
+          ]
+        }
+      ],
+      "dest": "api",
+      "disableGitFeatures": false,
+      "disableDefaultFilter": false
+    }
+  ],
+  "build": {
+    "content": [
+      {
+        "files": [
+          "api/**.yml",
+          "api/**.md",
+          "api/index.md"
+        ]
+      },
+      {
+        "files": [
+          "toc.yml",
+          "*.md"
+        ]
+      }
+    ],
+    "resource": [
+      {
+        "files": [
+          "images/**"
+        ]
+      }
+    ],
+    "overwrite": [
+      {
+        "files": [
+          "apidoc/**.md"
+        ],
+        "exclude": [
+          "obj/**",
+          "_site/**"
+        ]
+      }
+    ],
+    "dest": "_site",
+    "globalMetadataFiles": [],
+    "fileMetadataFiles": [],
+    "template": [
+      "default",
+      "templates/matthews"
+    ],
+    "postProcessors": [],
+    "markdownEngineName": "markdig",
+    "noLangKeyword": false,
+    "keepFileLink": false,
+    "cleanupCacheHistory": false,
+    "disableGitFeatures": false,
+    "globalMetadata": {
+      "_appTitle": "",
+      "_appFooter": "",
+      "_enableSearch": false,
+      "_enableNewTab": true,
+      "_appFaviconPath": "https://c.s-microsoft.com/favicon.ico?v2",
+      "_disableContribution": true
+    }    
+  }
+}
+
+
+
+
+
+
diff --git a/eng/common/docgeneration/templates/matthews/partials/affix.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/affix.tmpl.partial
@@ -0,0 +1,17 @@
+{{^_disableContribution}}
+<div class="contribution-panel mobile-hide">
+    {{#docurl}}
+    <a href="{{docurl}}" title="{{__global.improveThisDoc}}" class="fab btn-warning pull-right"><i class="glyphicon glyphicon-pencil"></i></a>
+    {{/docurl}}
+    {{#sourceurl}}
+    <a href="{{sourceurl}}" title="{{__global.viewSource}}" class="fab btn-info pull-right"><i class="fa fa-code"></i></a>
+    {{/sourceurl}}
+</div>
+{{/_disableContribution}}
+
+<div class="hidden-sm col-md-2" role="complementary">
+  <div class="sideaffix">
+    <nav class="bs-docs-sidebar hidden-print hidden-xs hidden-sm affix" id="affix">
+    </nav>
+  </div>
+</div>