Add azd ai dataset extension for Foundry datasets

.github/workflows/lint-ext-azure-ai-dataset.yml

@@ -0,0 +1,22 @@
+name: ext-azure-ai-dataset-ci
+
+on:
+  pull_request:
+    paths:
+      - "cli/azd/extensions/azure.ai.dataset/**"
+      - ".github/workflows/lint-ext-azure-ai-dataset.yml"
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write # required by reusable workflow lint-go.yml
+
+jobs:
+  lint:
+    uses: ./.github/workflows/lint-go.yml
+    with:
+      working-directory: cli/azd/extensions/azure.ai.dataset

cli/azd/.vscode/cspell.yaml

@@ -491,6 +491,9 @@ overrides:
     words:
       - forbidigo
       - Logf
+  - filename: extensions/azure.ai.dataset/README.md
+    words:
+      - CODEOWNERS
   - filename: .golangci.yaml
     words:
       - forbidigo

cli/azd/extensions/azure.ai.dataset/.gitignore

@@ -0,0 +1,5 @@
+# Test report written by ci-test.ps1 for the pipeline to publish.
+junitTestReport.xml
+
+# Debug log written when --debug or AZD_EXT_DEBUG is set.
+azd-ai-dataset-*.log

cli/azd/extensions/azure.ai.dataset/.golangci.yaml

@@ -0,0 +1,21 @@
+version: "2"
+
+linters:
+  default: none
+  enable:
+    - gosec
+    - lll
+    - unused
+    - errorlint
+  settings:
+    lll:
+      line-length: 220
+      tab-width: 4
+    gosec:
+      excludes:
+        - G204 # Subprocess launched with variable (bicep build invoked in tests)
+        - G304 # Potential file inclusion via variable
+
+formatters:
+  enable:
+    - gofmt

cli/azd/extensions/azure.ai.dataset/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Release History
+
+## 1.0.0-beta.6 (Unreleased)
+
+First release of the Foundry datasets extension.
+
+### Features Added
+
+- Register and manage Foundry datasets from the terminal: `create`, `update`,
+  `list`, `show`, `delete`, and `versions list`.
+- Publishes a local `.jsonl` file or folder as a versioned dataset, picking the
+  next version from what the project already carries.
+- Validates before sending: a malformed row, an empty dataset, a name the
+  service will not take, and a folder that could mean more than one dataset are
+  each refused locally.
+- Reads dataset content back, whether the service hands out a blob URI or the
+  container holding it.
+- `-o json` on every command, and `--no-prompt` for unattended use.

cli/azd/extensions/azure.ai.dataset/README.md

@@ -0,0 +1,63 @@
+# Foundry datasets (Beta)
+
+Register and version Foundry datasets from your terminal.
+
+```console
+$ azd extension install azure.ai.dataset
+$ azd ai dataset --help
+```
+
+A dataset is a general Foundry asset: evaluation needs one, and so do
+fine-tuning and other scenarios. That is why these commands live here rather
+than inside `azure.ai.evaluations`.
+
+## Commands
+
+| Command | What it does |
+|---|---|
+| `azd ai dataset create <name> --from-file <path>` | Register a dataset, publishing its first version |
+| `azd ai dataset update <name> --from-file <path>` | Publish a further version |
+| `azd ai dataset list` | List the project's datasets |
+| `azd ai dataset show <name>` | Show one dataset |
+| `azd ai dataset delete <name>` | Delete a dataset version |
+| `azd ai dataset versions list <name>` | List a dataset's versions |
+
+## Generating a dataset
+
+Generation is `azd ai eval generate`, in `azure.ai.evaluations`, and stays
+there: it writes the `datasets:` entry in `evals/azure.eval.yaml`, which is that
+extension's file. Splitting the two would leave a generated dataset registered
+with the service but absent from the configuration, so `azd up` would not
+reconcile it and no eval could name it.
+
+Once a file exists, `create` registers it here.
+
+## Project endpoint
+
+Every command resolves the Foundry project endpoint in this order:
+
+1. `--project-endpoint`
+2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment
+3. the host environment variable of the same name
+
+## Building
+
+```console
+$ go build ./...
+$ go test ./...
+```
+
+## TODO before release
+
+Both are files the azd extensions team owns, so they are not changed here:
+
+- [ ] **`cli/azd/extensions/registry.json`** — add the `azure.ai.dataset` entry.
+  Until it exists `azd extension install azure.ai.dataset` cannot resolve, so
+  the extension is only reachable through `azd x pack` + `azd x publish` into
+  the local source registry.
+- [ ] **`.github/CODEOWNERS`** — add `/cli/azd/extensions/azure.ai.dataset/`.
+  Every sibling Foundry extension has an entry; without one, PRs here get no
+  reviewer routing.
+- [ ] **`microsoft.foundry/extension.yaml`** — add the dependency, but only
+  after the registry entry lands. Declaring a dependency that cannot resolve
+  breaks installing the bundle.

cli/azd/extensions/azure.ai.dataset/build.ps1

@@ -0,0 +1,78 @@
+# Ensure script fails on any error
+$ErrorActionPreference = 'Stop'
+
+# Get the directory of the script
+$EXTENSION_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path
+
+# Change to the script directory
+Set-Location -Path $EXTENSION_DIR
+
+# Create a safe version of EXTENSION_ID replacing dots with dashes
+$EXTENSION_ID_SAFE = $env:EXTENSION_ID -replace '\.', '-'
+
+# Define output directory
+$OUTPUT_DIR = if ($env:OUTPUT_DIR) { $env:OUTPUT_DIR } else { Join-Path $EXTENSION_DIR "bin" }
+
+# Create output directory if it doesn't exist
+if (-not (Test-Path -Path $OUTPUT_DIR)) {
+    New-Item -ItemType Directory -Path $OUTPUT_DIR | Out-Null
+}
+
+# Get Git commit hash and build date
+$COMMIT = git rev-parse HEAD
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "Error: Failed to get git commit hash"
+    exit 1
+}
+$BUILD_DATE = (Get-Date -Format "yyyy-MM-ddTHH:mm:ssZ")
+
+# List of OS and architecture combinations
+if ($env:EXTENSION_PLATFORM) {
+    $PLATFORMS = @($env:EXTENSION_PLATFORM)
+}
+else {
+    $PLATFORMS = @(
+        "windows/amd64",
+        "windows/arm64",
+        "darwin/amd64",
+        "darwin/arm64",
+        "linux/amd64",
+        "linux/arm64"
+    )
+}
+
+$VERSION_PATH = "azureaidataset/internal/version"
+
+# Loop through platforms and build
+foreach ($PLATFORM in $PLATFORMS) {
+    $OS, $ARCH = $PLATFORM -split '/'
+
+    $OUTPUT_NAME = Join-Path $OUTPUT_DIR "$EXTENSION_ID_SAFE-$OS-$ARCH"
+
+    if ($OS -eq "windows") {
+        $OUTPUT_NAME += ".exe"
+    }
+
+    Write-Host "Building for $OS/$ARCH..."
+
+    # Delete the output file if it already exists
+    if (Test-Path -Path $OUTPUT_NAME) {
+        Remove-Item -Path $OUTPUT_NAME -Force
+    }
+
+    # Set environment variables for Go build
+    $env:GOOS = $OS
+    $env:GOARCH = $ARCH
+
+    go build `
+        -ldflags="-X '$VERSION_PATH.Version=$env:EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" `
+        -o $OUTPUT_NAME
+
+    if ($LASTEXITCODE -ne 0) {
+        Write-Host "An error occurred while building for $OS/$ARCH"
+        exit 1
+    }
+}
+
+Write-Host "Build completed successfully!"
+Write-Host "Binaries are located in the $OUTPUT_DIR directory."

cli/azd/extensions/azure.ai.dataset/build.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Get the directory of the script
+EXTENSION_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+# Change to the script directory
+cd "$EXTENSION_DIR" || exit
+
+# Create a safe version of EXTENSION_ID replacing dots with dashes
+EXTENSION_ID_SAFE="${EXTENSION_ID//./-}"
+
+# Define output directory
+OUTPUT_DIR="${OUTPUT_DIR:-$EXTENSION_DIR/bin}"
+
+# Create output and target directories if they don't exist
+mkdir -p "$OUTPUT_DIR"
+
+# Get Git commit hash and build date
+COMMIT=$(git rev-parse HEAD)
+BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+# List of OS and architecture combinations
+if [ -n "$EXTENSION_PLATFORM" ]; then
+    PLATFORMS=("$EXTENSION_PLATFORM")
+else
+    PLATFORMS=(
+        "windows/amd64"
+        "windows/arm64"
+        "darwin/amd64"
+        "darwin/arm64"
+        "linux/amd64"
+        "linux/arm64"
+    )
+fi
+
+VERSION_PATH="azureaidataset/internal/version"
+
+# Loop through platforms and build
+for PLATFORM in "${PLATFORMS[@]}"; do
+    OS=$(echo "$PLATFORM" | cut -d'/' -f1)
+    ARCH=$(echo "$PLATFORM" | cut -d'/' -f2)
+
+    OUTPUT_NAME="$OUTPUT_DIR/$EXTENSION_ID_SAFE-$OS-$ARCH"
+
+    if [ "$OS" = "windows" ]; then
+        OUTPUT_NAME+='.exe'
+    fi
+
+    echo "Building for $OS/$ARCH..."
+
+    # Delete the output file if it already exists
+    [ -f "$OUTPUT_NAME" ] && rm -f "$OUTPUT_NAME"
+
+    # Set environment variables for Go build
+    GOOS=$OS GOARCH=$ARCH go build \
+        -ldflags="-X '$VERSION_PATH.Version=$EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" \
+        -o "$OUTPUT_NAME"
+
+    if [ $? -ne 0 ]; then
+        echo "An error occurred while building for $OS/$ARCH"
+        exit 1
+    fi
+done
+
+echo "Build completed successfully!"
+echo "Binaries are located in the $OUTPUT_DIR directory."