diff --git a/application/single_app/config.py b/application/single_app/config.py index 4206520c9..05464869a 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.293" +VERSION = "0.261.294" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_action_manifest.py b/application/single_app/functions_action_manifest.py index bdf031bf9..32b9403a8 100644 --- a/application/single_app/functions_action_manifest.py +++ b/application/single_app/functions_action_manifest.py @@ -14,6 +14,42 @@ MCP_TYPE_ALIASES = frozenset({ "mcp", "mcpplugin", "modelcontextprotocol", "modelcontextprotocolplugin", }) +# Action types that run with the application identity against customer data and are +# therefore created only by administrators as global actions; users reach them through agents. +GLOBAL_ONLY_ACTION_TYPES = frozenset({"azure_files_index"}) +# Plugin class names behind GLOBAL_ONLY_ACTION_TYPES, keyed the way the Semantic Kernel loaders +# match an action type to a plugin class (see _loader_type_key). +_GLOBAL_ONLY_PLUGIN_CLASS_KEYS = frozenset({"azurefilesindexplugin"}) + + +class GlobalOnlyActionTypeError(PermissionError): + """An action type that only administrators may create, as a global action.""" + + code = "global_only_action_type" + public_message = "This action type is available only as an administrator-managed global action." + + def __init__(self): + super().__init__(self.public_message) + + +def _loader_type_key(action_type): + """Normalize a type exactly as the plugin loaders do before matching plugin class names.""" + text = re.sub(r"\s", "", str(action_type or "")) + return text.replace("_", "").replace("-", "").replace("plugin", "").lower() + + +def is_global_only_action_type(action_type): + """Return whether an action type may be created only as a global action. + + The plugin loaders run an action with the first plugin class whose normalized name contains + the normalized type, so aliases such as ``AzureFilesIndex`` or ``files_index`` reach a + global-only plugin class as well. A non-empty type that normalizes to nothing matches every + class and is treated the same way. + """ + if not str(action_type or "").strip(): + return False + key = _loader_type_key(action_type) + return not key or any(key in class_key for class_key in _GLOBAL_ONLY_PLUGIN_CLASS_KEYS) class McpConfigurationError(ValueError): diff --git a/application/single_app/functions_activity_logging.py b/application/single_app/functions_activity_logging.py index 61866bf4a..98d77b774 100644 --- a/application/single_app/functions_activity_logging.py +++ b/application/single_app/functions_activity_logging.py @@ -2134,6 +2134,62 @@ def log_file_sync_activity( debug_print(f"Warning: Failed to log File Sync activity: {str(e)}") +def log_azure_files_search_access(user_id: str, review: Dict[str, Any]) -> None: + """Record an Azure Files Search that withheld denied or unverifiable files. + + The review holds counts, reason codes, and a bounded list of withheld file paths. It never + holds file content or the user's query, and only administrators can read activity logs. + """ + normalized_user_id = coerce_activity_log_user_id(user_id) + now = datetime.utcnow().isoformat() + counts = (review or {}).get('counts') or {} + withheld = int(counts.get('denied_files') or 0) + int(counts.get('unverified_files') or 0) + try: + activity_record = { + 'id': str(uuid.uuid4()), + 'user_id': normalized_user_id, + 'activity_type': 'azure_files_search_access', + 'timestamp': now, + 'created_at': now, + 'action': 'results_withheld', + 'description': ( + f"Azure Files Search withheld {withheld} of {int(counts.get('files_evaluated') or 0)} files " + f"({int(counts.get('denied_files') or 0)} denied, {int(counts.get('unverified_files') or 0)} unverified)" + ), + 'conversation_id': (review or {}).get('conversation_id') or None, + 'agent': (review or {}).get('agent') or None, + 'action_context': { + 'action_id': (review or {}).get('action_id'), + 'action_name': (review or {}).get('action_name'), + 'display_name': (review or {}).get('display_name'), + 'search_service': (review or {}).get('search_service'), + 'index_name': (review or {}).get('index_name'), + }, + 'additional_context': { + 'status': (review or {}).get('status'), + 'permission_mode': (review or {}).get('permission_mode'), + 'share_access_check': (review or {}).get('share_access_check'), + 'counts': counts, + 'reasons': (review or {}).get('reasons') or {}, + 'withheld_files': (review or {}).get('withheld_files') or [], + 'withheld_files_truncated': bool((review or {}).get('withheld_files_truncated')), + 'duration_ms': (review or {}).get('duration_ms'), + }, + } + cosmos_activity_logs_container.create_item(body=activity_record) + debug_print("[AZURE_FILES_SEARCH] Access review logged.") + except Exception as e: + log_event( + message="[AZURE_FILES_SEARCH] Error logging access review.", + extra={ + 'user_id': normalized_user_id, + 'action_id': (review or {}).get('action_id'), + 'exception_type': type(e).__name__, + }, + level=logging.ERROR + ) + + def log_governance_change( admin_user_id: str, admin_email: str, diff --git a/application/single_app/functions_azure_endpoint_validation.py b/application/single_app/functions_azure_endpoint_validation.py index 3ab64a4f5..b855d51b5 100644 --- a/application/single_app/functions_azure_endpoint_validation.py +++ b/application/single_app/functions_azure_endpoint_validation.py @@ -41,6 +41,12 @@ "api.loganalytics.us", "api.loganalytics.azure.cn", ) +# Azure AI Search service suffixes for the public, US Government, and China clouds. +AZURE_SEARCH_ENDPOINT_SUFFIXES = ( + "search.windows.net", + "search.azure.us", + "search.azure.cn", +) # Mirrors azure.identity.AzureAuthorityHosts so a caller cannot select a token authority. AZURE_ENTRA_AUTHORITY_HOSTS = ( "login.microsoftonline.com", @@ -106,6 +112,14 @@ "The application identity can be used only with an HTTPS Azure AI endpoint in this " "cloud, such as https://resource.openai.azure.com" ) +AZURE_SEARCH_ENDPOINT_ERROR = ( + "Azure AI Search actions require an HTTPS Azure AI Search service endpoint such as " + "https://service.search.windows.net" +) +AZURE_FILE_ENDPOINT_ERROR = ( + "Azure Files requires an HTTPS Azure Files service endpoint such as " + "https://account.file.core.windows.net" +) def _normalize_endpoint_text(value: Any) -> str: @@ -243,6 +257,29 @@ def validate_azure_queue_endpoint(value: Any) -> str: return _validate_storage_endpoint(value, AZURE_QUEUE_SERVICE_LABEL, AZURE_QUEUE_ENDPOINT_ERROR) +def validate_azure_file_endpoint(value: Any) -> str: + """Return a canonical Azure Files service origin, or raise ValueError.""" + return _validate_storage_endpoint(value, AZURE_FILE_SERVICE_LABEL, AZURE_FILE_ENDPOINT_ERROR) + + +def validate_azure_search_endpoint(value: Any) -> str: + """Return a canonical Azure AI Search service origin, or raise ValueError.""" + _, hostname = parse_azure_https_endpoint( + value, + AZURE_SEARCH_ENDPOINT_ERROR, + allow_default_port=True, + ) + service_name, endpoint_suffix = _match_endpoint_suffix( + hostname, + "", + AZURE_SEARCH_ENDPOINT_SUFFIXES, + AZURE_SEARCH_ENDPOINT_ERROR, + ) + if not DNS_LABEL_PATTERN.match(service_name) or not 2 <= len(service_name) <= 60: + raise ValueError(AZURE_SEARCH_ENDPOINT_ERROR) + return f"https://{service_name}.{endpoint_suffix}" + + def validate_azure_cosmos_endpoint(value: Any) -> str: """Return a canonical Azure Cosmos DB origin, or raise ValueError.""" _, hostname = parse_azure_https_endpoint( diff --git a/application/single_app/functions_azure_files_access.py b/application/single_app/functions_azure_files_access.py new file mode 100644 index 000000000..7d92079c5 --- /dev/null +++ b/application/single_app/functions_azure_files_access.py @@ -0,0 +1,470 @@ +# functions_azure_files_access.py +"""Resolve who a signed-in user is to Azure Files, and whether a share admits them. + +The Azure Files Search action evaluates each candidate file's NTFS DACL itself, because +Azure Files OAuth reads use backup intent and bypass file permissions. This module supplies +the identity side of that evaluation: + +* the Windows security identifiers (SIDs) the user and their groups carry, from Microsoft + Graph (``securityIdentifier`` and ``onPremisesSecurityIdentifier``, plus the SID derived + from each Entra object ID); +* whether another principal named in an ACL is a different directory object, so its entries + can be ignored rather than treated as unknown; +* whether the user has share-level access, from the storage account's default share-level + permission and the user's Azure role assignments on the share. + +Every lookup failure resolves to "unknown", which callers treat as unverified. +""" + +import logging +import re +import struct +import threading +import time +import uuid +from dataclasses import dataclass +from typing import Any, Callable, Dict, FrozenSet, Iterable, Optional, Tuple +from urllib.parse import quote + +import requests + +from functions_appinsights import log_event +from functions_azure_files_acl import MEMBER, NOT_MEMBER, UNKNOWN, normalize_sid + +GRAPH_REQUEST_TIMEOUT_SECONDS = 10 +ARM_REQUEST_TIMEOUT_SECONDS = 10 +PRINCIPALS_CACHE_TTL_SECONDS = 600 +SID_LOOKUP_CACHE_TTL_SECONDS = 3600 +SHARE_ACCESS_CACHE_TTL_SECONDS = 600 +ROLE_DEFINITION_CACHE_TTL_SECONDS = 3600 +STORAGE_ACCOUNT_CACHE_TTL_SECONDS = 600 +CACHE_MAX_ENTRIES = 5000 +MAX_GROUP_PAGES = 25 +MAX_ROLE_ASSIGNMENT_PAGES = 10 + +STORAGE_ACCOUNT_API_VERSION = "2023-05-01" +AUTHORIZATION_API_VERSION = "2022-04-01" +FILE_SHARE_READ_DATA_ACTION = "Microsoft.Storage/storageAccounts/fileServices/fileshares/files/read" +# Default share-level permissions that let every authenticated identity read a share. +READ_GRANTING_DEFAULT_SHARE_PERMISSIONS = frozenset({ + "storagefiledatasmbsharereader", + "storagefiledatasmbsharecontributor", + "storagefiledatasmbshareelevatedcontributor", +}) + +REASON_IDENTITY_UNAVAILABLE = "identity_unavailable" +REASON_SHARE_ACCESS_DENIED = "share_access_denied" +REASON_SHARE_ACCESS_UNKNOWN = "share_access_unknown" +REASON_SHARE_DEFAULT_PERMISSION = "share_default_permission" +REASON_SHARE_ROLE_ASSIGNMENT = "share_role_assignment" + +_STORAGE_ACCOUNT_RESOURCE_ID_PATTERN = re.compile( + r"^/subscriptions/([0-9a-fA-F-]{36})/resourceGroups/([-\w._()]{1,90})" + r"/providers/Microsoft\.Storage/storageAccounts/([a-z0-9]{3,24})$", + re.IGNORECASE, +) +_SHARE_NAME_PATTERN = re.compile(r"^[a-z0-9](?:[a-z0-9]|-(?=[a-z0-9])){2,62}$") +_ROLE_DEFINITION_ID_PATTERN = re.compile( + r"^(/subscriptions/[0-9a-fA-F-]{36})?/providers/Microsoft\.Authorization/roleDefinitions/[0-9a-fA-F-]{36}$", + re.IGNORECASE, +) + +HttpGet = Callable[..., Any] + + +class AzureFilesIdentityError(RuntimeError): + """The user's directory identity could not be resolved; access stays unverified.""" + + def __init__(self, reason: str): + self.reason = reason + super().__init__(reason) + + +class _TtlCache: + """A small thread-safe cache with per-entry expiry and a bounded size.""" + + def __init__(self, ttl_seconds: int, max_entries: int = CACHE_MAX_ENTRIES): + self._ttl_seconds = ttl_seconds + self._max_entries = max_entries + self._entries: Dict[Any, Tuple[float, Any]] = {} + self._lock = threading.Lock() + + def get(self, key: Any) -> Any: + with self._lock: + entry = self._entries.get(key) + if entry is None: + return None + expires_at, value = entry + if expires_at < time.monotonic(): + self._entries.pop(key, None) + return None + return value + + def set(self, key: Any, value: Any, ttl_seconds: Optional[int] = None) -> None: + with self._lock: + if len(self._entries) >= self._max_entries: + now = time.monotonic() + for stale_key in [k for k, (expires_at, _) in self._entries.items() if expires_at < now]: + self._entries.pop(stale_key, None) + if len(self._entries) >= self._max_entries: + self._entries.pop(next(iter(self._entries))) + self._entries[key] = (time.monotonic() + (ttl_seconds or self._ttl_seconds), value) + + def clear(self) -> None: + with self._lock: + self._entries.clear() + + +_principals_cache = _TtlCache(PRINCIPALS_CACHE_TTL_SECONDS) +_sid_lookup_cache = _TtlCache(SID_LOOKUP_CACHE_TTL_SECONDS) +_share_access_cache = _TtlCache(SHARE_ACCESS_CACHE_TTL_SECONDS) +_role_definition_cache = _TtlCache(ROLE_DEFINITION_CACHE_TTL_SECONDS) +_storage_account_cache = _TtlCache(STORAGE_ACCOUNT_CACHE_TTL_SECONDS) + + +def clear_azure_files_access_caches() -> None: + """Drop every cached identity, SID, and share decision.""" + for cache in (_principals_cache, _sid_lookup_cache, _share_access_cache, _role_definition_cache, _storage_account_cache): + cache.clear() + + +@dataclass(frozen=True) +class UserPrincipals: + user_id: str + sids: FrozenSet[str] + group_ids: FrozenSet[str] + + +@dataclass(frozen=True) +class StorageAccountReference: + resource_id: str + subscription_id: str + resource_group: str + account_name: str + + +def parse_storage_account_resource_id(value: Any) -> StorageAccountReference: + """Validate an Azure Storage account resource ID and return its parts.""" + text = str(value or "").strip().rstrip("/") + match = _STORAGE_ACCOUNT_RESOURCE_ID_PATTERN.match(text) + if not match: + raise ValueError( + "Storage accounts must be Azure resource IDs such as " + "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts/." + ) + subscription_id, resource_group, account_name = match.groups() + account_name = account_name.lower() + canonical = ( + f"/subscriptions/{subscription_id.lower()}/resourceGroups/{resource_group}" + f"/providers/Microsoft.Storage/storageAccounts/{account_name}" + ) + return StorageAccountReference(canonical, subscription_id.lower(), resource_group, account_name) + + +def is_valid_share_name(value: Any) -> bool: + return bool(_SHARE_NAME_PATTERN.match(str(value or ""))) + + +def entra_object_id_to_sid(object_id: Any) -> str: + """Return the ``S-1-12-1-...`` SID Microsoft Entra derives from an object ID.""" + try: + object_uuid = uuid.UUID(str(object_id or "")) + except ValueError: + return "" + parts = struct.unpack(" str: + """Return the Entra object ID behind an ``S-1-12-1-a-b-c-d`` SID, or ``""``.""" + match = re.match(r"^S-1-12-1-(\d+)-(\d+)-(\d+)-(\d+)$", normalize_sid(sid)) + if not match: + return "" + parts = [int(part) for part in match.groups()] + if any(part > 0xFFFFFFFF for part in parts): + return "" + return str(uuid.UUID(bytes_le=struct.pack(" Any: + headers = {"Authorization": f"Bearer {access_token}", "Accept": "application/json"} + if consistency: + headers["ConsistencyLevel"] = "eventual" + return http_get(url, headers=headers, timeout=GRAPH_REQUEST_TIMEOUT_SECONDS) + + +def _response_json(response: Any) -> Dict[str, Any]: + try: + payload = response.json() + except ValueError: + return {} + return payload if isinstance(payload, dict) else {} + + +def _principal_sids(directory_object: Dict[str, Any]) -> Iterable[str]: + yield normalize_sid(directory_object.get("securityIdentifier")) + yield normalize_sid(directory_object.get("onPremisesSecurityIdentifier")) + yield entra_object_id_to_sid(directory_object.get("id")) + + +def get_user_principals( + user_id: str, + access_token: str, + graph_base_url: str, + http_get: HttpGet = requests.get, +) -> UserPrincipals: + """Return the SIDs the signed-in user and every group they belong to carry. + + The delegated token must belong to ``user_id``; a token for anyone else is refused. + Group membership is transitive. An incomplete membership list raises, because a + missing group could hide an allow or a deny. + """ + normalized_user_id = str(user_id or "").strip().lower() + if not normalized_user_id or not access_token: + raise AzureFilesIdentityError(REASON_IDENTITY_UNAVAILABLE) + cached = _principals_cache.get(normalized_user_id) + if cached is not None: + return cached + + base_url = graph_base_url.rstrip("/") + try: + me_response = _graph_get( + f"{base_url}/me?$select=id,securityIdentifier,onPremisesSecurityIdentifier", + access_token, + http_get, + ) + if me_response.status_code != 200: + raise AzureFilesIdentityError(REASON_IDENTITY_UNAVAILABLE) + me = _response_json(me_response) + if str(me.get("id") or "").lower() != normalized_user_id: + raise AzureFilesIdentityError(REASON_IDENTITY_UNAVAILABLE) + + sids = set(_principal_sids(me)) + group_ids = set() + next_url = ( + f"{base_url}/me/transitiveMemberOf/microsoft.graph.group" + "?$select=id,securityIdentifier,onPremisesSecurityIdentifier&$top=999" + ) + for _ in range(MAX_GROUP_PAGES): + response = _graph_get(next_url, access_token, http_get) + if response.status_code != 200: + raise AzureFilesIdentityError(REASON_IDENTITY_UNAVAILABLE) + payload = _response_json(response) + for group in payload.get("value") or []: + if isinstance(group, dict) and group.get("id"): + group_ids.add(str(group["id"]).lower()) + sids.update(_principal_sids(group)) + next_url = payload.get("@odata.nextLink") + if not next_url: + break + else: + raise AzureFilesIdentityError(REASON_IDENTITY_UNAVAILABLE) + except AzureFilesIdentityError: + raise + except Exception as error: + log_event( + "[AZURE_FILES_SEARCH] Could not resolve the user's directory principals.", + level=logging.WARNING, + extra={"exception_type": type(error).__name__}, + ) + raise AzureFilesIdentityError(REASON_IDENTITY_UNAVAILABLE) from error + + principals = UserPrincipals( + user_id=normalized_user_id, + sids=frozenset(sid for sid in sids if sid), + group_ids=frozenset(group_ids), + ) + _principals_cache.set(normalized_user_id, principals) + return principals + + +def classify_foreign_sid( + sid: str, + access_token: str, + graph_base_url: str, + http_get: HttpGet = requests.get, +) -> str: + """Return ``not_member`` when another directory object owns ``sid``, else ``unknown``. + + Called only for SIDs that are not the user's own or one of their groups'. A SID that + belongs to another Entra user, or to a group whose membership is fully known, does not + apply to the user. A SID the directory cannot place (an AD-only group, a local or + deleted account, another tenant) stays unknown. + """ + normalized = normalize_sid(sid) + if not normalized or not access_token: + return UNKNOWN + cached = _sid_lookup_cache.get(normalized) + if cached is not None: + return cached + + base_url = graph_base_url.rstrip("/") + state = UNKNOWN + cacheable = True + try: + object_id = entra_sid_to_object_id(normalized) + if object_id: + for collection in ("groups", "users"): + response = _graph_get(f"{base_url}/{collection}/{object_id}?$select=id", access_token, http_get) + if response.status_code == 200: + state = NOT_MEMBER + break + if response.status_code != 404: + cacheable = False + break + else: + for collection, attribute in ( + ("groups", "securityIdentifier"), + ("groups", "onPremisesSecurityIdentifier"), + ("users", "onPremisesSecurityIdentifier"), + ): + response = _graph_get( + f"{base_url}/{collection}?$filter={attribute}%20eq%20'{quote(normalized, safe='-')}'" + "&$select=id&$top=1&$count=true", + access_token, + http_get, + consistency=True, + ) + if response.status_code != 200: + cacheable = False + continue + if _response_json(response).get("value"): + state = NOT_MEMBER + break + except Exception as error: + log_event( + "[AZURE_FILES_SEARCH] Could not look up a principal named in a file ACL.", + level=logging.WARNING, + extra={"exception_type": type(error).__name__}, + ) + return UNKNOWN + + if state == NOT_MEMBER or cacheable: + _sid_lookup_cache.set(normalized, state) + return state + + +def _action_matches(pattern: str, action: str) -> bool: + regex = "^" + re.escape(pattern).replace(r"\*", ".*") + "$" + return re.match(regex, action, re.IGNORECASE) is not None + + +def role_definition_grants_file_read(role_definition: Dict[str, Any]) -> bool: + """Return whether a role definition's data actions include reading file share data.""" + for permission in (role_definition.get("properties") or {}).get("permissions") or []: + data_actions = permission.get("dataActions") or [] + not_data_actions = permission.get("notDataActions") or [] + if any(_action_matches(pattern, FILE_SHARE_READ_DATA_ACTION) for pattern in data_actions) and not any( + _action_matches(pattern, FILE_SHARE_READ_DATA_ACTION) for pattern in not_data_actions + ): + return True + return False + + +def _arm_get(url: str, access_token: str, http_get: HttpGet) -> Any: + return http_get( + url, + headers={"Authorization": f"Bearer {access_token}", "Accept": "application/json"}, + timeout=ARM_REQUEST_TIMEOUT_SECONDS, + ) + + +def _default_share_permission(account: StorageAccountReference, arm_base_url: str, arm_token: str, http_get: HttpGet) -> str: + cached = _storage_account_cache.get(account.resource_id) + if cached is not None: + return cached + response = _arm_get(f"{arm_base_url}{account.resource_id}?api-version={STORAGE_ACCOUNT_API_VERSION}", arm_token, http_get) + if response.status_code != 200: + raise RuntimeError("storage_account_unreadable") + identity_auth = (_response_json(response).get("properties") or {}).get("azureFilesIdentityBasedAuthentication") or {} + permission = str(identity_auth.get("defaultSharePermission") or "None").replace("_", "").lower() + _storage_account_cache.set(account.resource_id, permission) + return permission + + +def _role_definition(role_definition_id: str, arm_base_url: str, arm_token: str, http_get: HttpGet) -> Dict[str, Any]: + cache_key = role_definition_id.lower() + cached = _role_definition_cache.get(cache_key) + if cached is not None: + return cached + if not _ROLE_DEFINITION_ID_PATTERN.match(role_definition_id): + raise RuntimeError("role_definition_invalid") + response = _arm_get(f"{arm_base_url}{role_definition_id}?api-version={AUTHORIZATION_API_VERSION}", arm_token, http_get) + if response.status_code != 200: + raise RuntimeError("role_definition_unreadable") + definition = _response_json(response) + _role_definition_cache.set(cache_key, definition) + return definition + + +def check_share_level_access( + user_id: str, + account: StorageAccountReference, + share_name: str, + arm_base_url: str, + get_arm_token: Callable[[], str], + http_get: HttpGet = requests.get, +) -> Tuple[str, str]: + """Return ``(state, reason)`` for whether share-level permissions admit the user. + + A default share-level permission that grants read admits every authenticated identity. + Otherwise the user's role assignments at, above, or (through groups) for the share are + checked for a role whose data actions include reading files. Role assignments carrying + conditions are not evaluated and leave access unknown. + """ + normalized_user_id = str(user_id or "").strip().lower() + if not normalized_user_id or not is_valid_share_name(share_name): + return UNKNOWN, REASON_SHARE_ACCESS_UNKNOWN + cache_key = (normalized_user_id, account.resource_id, share_name) + cached = _share_access_cache.get(cache_key) + if cached is not None: + return cached + + arm_base_url = arm_base_url.rstrip("/") + try: + arm_token = get_arm_token() + if _default_share_permission(account, arm_base_url, arm_token, http_get) in READ_GRANTING_DEFAULT_SHARE_PERMISSIONS: + decision = (MEMBER, REASON_SHARE_DEFAULT_PERMISSION) + _share_access_cache.set(cache_key, decision) + return decision + + share_scope = f"{account.resource_id}/fileServices/default/fileshares/{share_name}" + next_url = ( + f"{arm_base_url}{share_scope}/providers/Microsoft.Authorization/roleAssignments" + f"?api-version={AUTHORIZATION_API_VERSION}&$filter=assignedTo('{normalized_user_id}')" + ) + conditional_grant = False + for _ in range(MAX_ROLE_ASSIGNMENT_PAGES): + response = _arm_get(next_url, arm_token, http_get) + if response.status_code != 200: + raise RuntimeError("role_assignments_unreadable") + payload = _response_json(response) + for assignment in payload.get("value") or []: + properties = (assignment or {}).get("properties") or {} + role_definition_id = str(properties.get("roleDefinitionId") or "") + if not role_definition_id: + continue + if not role_definition_grants_file_read(_role_definition(role_definition_id, arm_base_url, arm_token, http_get)): + continue + if properties.get("condition"): + conditional_grant = True + continue + decision = (MEMBER, REASON_SHARE_ROLE_ASSIGNMENT) + _share_access_cache.set(cache_key, decision) + return decision + next_url = payload.get("nextLink") + if not next_url: + break + else: + raise RuntimeError("role_assignments_incomplete") + except Exception as error: + log_event( + "[AZURE_FILES_SEARCH] Could not check share-level permissions.", + level=logging.WARNING, + extra={"exception_type": type(error).__name__, "storage_account": account.account_name, "share": share_name}, + ) + return UNKNOWN, REASON_SHARE_ACCESS_UNKNOWN + + decision = (UNKNOWN, REASON_SHARE_ACCESS_UNKNOWN) if conditional_grant else (NOT_MEMBER, REASON_SHARE_ACCESS_DENIED) + _share_access_cache.set(cache_key, decision) + return decision diff --git a/application/single_app/functions_azure_files_acl.py b/application/single_app/functions_azure_files_acl.py new file mode 100644 index 000000000..259a38168 --- /dev/null +++ b/application/single_app/functions_azure_files_acl.py @@ -0,0 +1,533 @@ +# functions_azure_files_acl.py +"""Parse Azure Files security descriptors (SDDL) and decide whether a user can read a file. + +Azure Files OAuth reads use backup intent, which bypasses NTFS permissions, so SimpleChat +evaluates each file's DACL itself before any of the file's content reaches a model. + +Evaluation mirrors Windows' ordered DACL check for FILE_READ_DATA and fails closed: when +membership in a referenced principal, a conditional or object ACE, or rights granted only +through generic bits could change the answer, the outcome is ``unverified`` rather than +``allowed``. This module performs no I/O; callers supply a membership function. +""" + +import re +from dataclasses import dataclass +from typing import Callable, FrozenSet, Iterable, Optional, Tuple + +OUTCOME_ALLOWED = "allowed" +OUTCOME_DENIED = "denied" +OUTCOME_UNVERIFIED = "unverified" + +REASON_ACL_ALLOW = "acl_allow" +REASON_ACL_NULL_DACL = "acl_null_dacl" +REASON_ACL_EXPLICIT_DENY = "acl_explicit_deny" +REASON_ACL_NO_ALLOW = "acl_no_allow" +REASON_ACL_EMPTY = "acl_empty" +REASON_SID_UNRESOLVED = "sid_unresolved" +REASON_ACL_UNSUPPORTED = "acl_unsupported" +REASON_ACL_PARSE_ERROR = "acl_parse_error" +REASON_ACL_MISSING = "acl_missing" + +MEMBER = "member" +NOT_MEMBER = "not_member" +UNKNOWN = "unknown" + +FILE_READ_DATA = 0x1 +GENERIC_ALL = 0x10000000 +GENERIC_EXECUTE = 0x20000000 +GENERIC_WRITE = 0x40000000 +GENERIC_READ = 0x80000000 +# GENERIC_READ and GENERIC_ALL map to rights that include FILE_READ_DATA for files. +GENERIC_RIGHTS_INCLUDING_READ = GENERIC_READ | GENERIC_ALL + +MAX_SDDL_LENGTH = 65536 +MAX_ACE_COUNT = 1820 + +EVERYONE_SID = "S-1-1-0" +CREATOR_OWNER_SID = "S-1-3-0" +CREATOR_GROUP_SID = "S-1-3-1" +OWNER_RIGHTS_SID = "S-1-3-4" +NETWORK_SID = "S-1-5-2" +AUTHENTICATED_USERS_SID = "S-1-5-11" +BUILTIN_USERS_SID = "S-1-5-32-545" +DOMAIN_USERS_RID = 513 + +SDDL_RIGHTS_ALIASES = { + "GA": GENERIC_ALL, + "GR": GENERIC_READ, + "GW": GENERIC_WRITE, + "GX": GENERIC_EXECUTE, + "RC": 0x00020000, + "SD": 0x00010000, + "WD": 0x00040000, + "WO": 0x00080000, + "RP": 0x00000010, + "WP": 0x00000020, + "CC": 0x00000001, + "DC": 0x00000002, + "LC": 0x00000004, + "SW": 0x00000008, + "LO": 0x00000080, + "DT": 0x00000040, + "CR": 0x00000100, + "FA": 0x001F01FF, + "FR": 0x00120089, + "FW": 0x00120116, + "FX": 0x001200A0, + "KA": 0x000F003F, + "KR": 0x00020019, + "KW": 0x00020006, + "KX": 0x00020019, +} + +# Machine- and domain-independent SDDL SID aliases. +SDDL_SID_ALIASES = { + "WD": EVERYONE_SID, + "CO": CREATOR_OWNER_SID, + "CG": CREATOR_GROUP_SID, + "OW": OWNER_RIGHTS_SID, + "NU": NETWORK_SID, + "BT": "S-1-5-3", + "IU": "S-1-5-4", + "SU": "S-1-5-6", + "AN": "S-1-5-7", + "ED": "S-1-5-9", + "PS": "S-1-5-10", + "AU": AUTHENTICATED_USERS_SID, + "RC": "S-1-5-12", + "SY": "S-1-5-18", + "LS": "S-1-5-19", + "NS": "S-1-5-20", + "WR": "S-1-5-33", + "BA": "S-1-5-32-544", + "BU": BUILTIN_USERS_SID, + "BG": "S-1-5-32-546", + "PU": "S-1-5-32-547", + "AO": "S-1-5-32-548", + "SO": "S-1-5-32-549", + "PO": "S-1-5-32-550", + "BO": "S-1-5-32-551", + "RE": "S-1-5-32-552", + "RU": "S-1-5-32-554", + "RD": "S-1-5-32-555", + "NO": "S-1-5-32-556", + "MU": "S-1-5-32-558", + "CD": "S-1-5-32-574", + "AC": "S-1-15-2-1", + "LW": "S-1-16-4096", + "ME": "S-1-16-8192", + "HI": "S-1-16-12288", + "SI": "S-1-16-16384", +} +# Aliases relative to a domain or machine that the SDDL string does not name. They are kept +# as ``alias:`` because the SID behind them cannot be reconstructed here. +SDDL_RELATIVE_SID_ALIASES = {"DA", "DU", "DG", "DC", "DD", "CA", "EA", "SA", "PA", "RS", "RO", "LA", "LG", "CN", "AP", "KA", "EK"} +# Relative aliases for administrator, service, machine and guest principals. SimpleChat does not +# treat its signed-in users as members of these. +PRIVILEGED_RELATIVE_SID_ALIASES = {"DA", "DG", "DC", "DD", "CA", "EA", "SA", "PA", "RS", "RO", "LA", "LG", "CN", "KA", "EK"} + +# Principals present in a network (SMB) logon session for any authenticated user. +ALWAYS_MEMBER_SIDS = frozenset({EVERYONE_SID, AUTHENTICATED_USERS_SID, NETWORK_SID}) +# System, service, placeholder, logon-type, integrity and administrator principals. A user +# reaching a share through SimpleChat is never treated as one of these. +NEVER_MEMBER_SIDS = frozenset({ + CREATOR_OWNER_SID, + CREATOR_GROUP_SID, + "S-1-5-3", + "S-1-5-4", + "S-1-5-6", + "S-1-5-7", + "S-1-5-9", + "S-1-5-10", + "S-1-5-12", + "S-1-5-18", + "S-1-5-19", + "S-1-5-20", + "S-1-5-33", + "S-1-5-32-544", + "S-1-5-32-546", + "S-1-5-32-547", + "S-1-5-32-548", + "S-1-5-32-549", + "S-1-5-32-550", + "S-1-5-32-551", + "S-1-5-32-552", + "S-1-5-32-554", + "S-1-5-32-555", + "S-1-5-32-556", + "S-1-5-32-558", + "S-1-5-32-574", + "S-1-15-2-1", + "S-1-16-4096", + "S-1-16-8192", + "S-1-16-12288", + "S-1-16-16384", +}) +# Well-known domain RIDs for built-in administrator, guest, machine and service accounts. +PRIVILEGED_DOMAIN_RIDS = frozenset({498, 500, 501, 502, 512, 514, 515, 516, 517, 518, 519, 520, 521, 553}) + +ACE_POLARITY = { + "A": "allow", + "D": "deny", + "OA": "allow", + "OD": "deny", + "XA": "allow", + "XD": "deny", + "ZA": "allow", +} +# Audit, alarm, label and attribute ACEs never grant or deny access. +NON_ACCESS_ACE_TYPES = frozenset({"AU", "AL", "OU", "OL", "ML", "XU", "SP", "RA"}) +SUPPORTED_ACE_TYPES = frozenset({"A", "D"}) +ACE_FLAG_TOKENS = frozenset({"CI", "OI", "NP", "IO", "ID", "SA", "FA", "TP", "CR"}) +DACL_FLAG_TOKENS = ("NO_ACCESS_CONTROL", "AR", "AI", "P") + +_SID_PATTERN = re.compile(r"^S-1-\d+(-\d+)+$", re.IGNORECASE) +_DOMAIN_SID_PATTERN = re.compile(r"^S-1-5-21-(\d+)-(\d+)-(\d+)-(\d+)$") +_COMPONENT_PATTERN = re.compile(r"[OGDS]:") + + +class SddlParseError(ValueError): + """The SDDL string could not be parsed; the caller must treat the file as unverified.""" + + +@dataclass(frozen=True) +class AccessControlEntry: + ace_type: str + flags: FrozenSet[str] + mask: int + sid: str + + +@dataclass(frozen=True) +class SecurityDescriptor: + owner_sid: str + group_sid: str + dacl_present: bool + dacl_null: bool + dacl_flags: FrozenSet[str] + dacl: Tuple[AccessControlEntry, ...] + + +@dataclass(frozen=True) +class AccessDecision: + outcome: str + reason: str + unresolved_sids: Tuple[str, ...] = () + unsupported_aces: Tuple[str, ...] = () + + @property + def allowed(self) -> bool: + return self.outcome == OUTCOME_ALLOWED + + +def normalize_sid(value: str) -> str: + """Return a SID in canonical ``S-1-...`` form, or ``""`` when it is not a SID.""" + text = str(value or "").strip() + if not _SID_PATTERN.match(text): + return "" + return f"S{text[1:]}" + + +def domain_sid_and_rid(sid: str) -> Tuple[str, Optional[int]]: + """Split ``S-1-5-21-a-b-c-rid`` into its domain SID and relative ID.""" + match = _DOMAIN_SID_PATTERN.match(sid or "") + if not match: + return "", None + return f"S-1-5-21-{match.group(1)}-{match.group(2)}-{match.group(3)}", int(match.group(4)) + + +def _resolve_sid_token(token: str) -> str: + text = str(token or "").strip() + sid = normalize_sid(text) + if sid: + return sid + alias = text.upper() + if alias in SDDL_SID_ALIASES: + return SDDL_SID_ALIASES[alias] + if alias in SDDL_RELATIVE_SID_ALIASES: + return f"alias:{alias}" + raise SddlParseError("The security descriptor contains an unrecognized principal.") + + +def _parse_rights(text: str) -> int: + value = str(text or "").strip() + if not value: + raise SddlParseError("An ACE has no access rights.") + if value[:2].lower() == "0x": + try: + return int(value, 16) + except ValueError as error: + raise SddlParseError("An ACE has invalid access rights.") from error + if value.isdigit(): + return int(value) + if len(value) % 2: + raise SddlParseError("An ACE has invalid access rights.") + mask = 0 + for index in range(0, len(value), 2): + alias = value[index:index + 2].upper() + if alias not in SDDL_RIGHTS_ALIASES: + raise SddlParseError("An ACE has an unrecognized access right.") + mask |= SDDL_RIGHTS_ALIASES[alias] + return mask + + +def _parse_ace_flags(text: str) -> FrozenSet[str]: + value = str(text or "").strip().upper() + if len(value) % 2: + raise SddlParseError("An ACE has invalid flags.") + flags = set() + for index in range(0, len(value), 2): + flag = value[index:index + 2] + if flag not in ACE_FLAG_TOKENS: + raise SddlParseError("An ACE has an unrecognized flag.") + flags.add(flag) + return frozenset(flags) + + +def _parse_ace(text: str) -> AccessControlEntry: + parts = text.split(";", 5) + if len(parts) != 6: + raise SddlParseError("An ACE is malformed.") + ace_type = parts[0].strip().upper() + if ace_type not in ACE_POLARITY and ace_type not in NON_ACCESS_ACE_TYPES: + raise SddlParseError("An ACE has an unrecognized type.") + # A conditional or resource-attribute ACE carries an expression after the SID. + sid_token = parts[5].split(";", 1)[0] + return AccessControlEntry( + ace_type=ace_type, + flags=_parse_ace_flags(parts[1]), + mask=_parse_rights(parts[2]), + sid=_resolve_sid_token(sid_token), + ) + + +def _split_aces(text: str, start: int) -> Tuple[Tuple[str, ...], int]: + aces = [] + index = start + while index < len(text) and text[index] == "(": + depth = 0 + in_quote = False + cursor = index + while cursor < len(text): + char = text[cursor] + if in_quote: + if char == '"': + in_quote = False + elif char == '"': + in_quote = True + elif char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + break + cursor += 1 + if cursor >= len(text): + raise SddlParseError("An ACE is not terminated.") + aces.append(text[index + 1:cursor]) + if len(aces) > MAX_ACE_COUNT: + raise SddlParseError("The security descriptor has too many ACEs.") + index = cursor + 1 + return tuple(aces), index + + +def _parse_acl_flags(text: str) -> Tuple[FrozenSet[str], bool]: + remaining = text.strip().upper() + flags = set() + null_acl = False + while remaining: + for token in DACL_FLAG_TOKENS: + if remaining.startswith(token): + if token == "NO_ACCESS_CONTROL": + null_acl = True + else: + flags.add(token) + remaining = remaining[len(token):] + break + else: + raise SddlParseError("The ACL has unrecognized flags.") + return frozenset(flags), null_acl + + +def parse_sddl(sddl: str) -> SecurityDescriptor: + """Parse an SDDL security descriptor string into its owner, group and DACL.""" + text = str(sddl or "").strip() + if not text: + raise SddlParseError("The security descriptor is empty.") + if len(text) > MAX_SDDL_LENGTH: + raise SddlParseError("The security descriptor is too large.") + + owner_sid = "" + group_sid = "" + dacl_present = False + dacl_null = False + dacl_flags: FrozenSet[str] = frozenset() + dacl: Tuple[AccessControlEntry, ...] = () + seen = set() + index = 0 + while index < len(text): + if index + 1 >= len(text) or text[index] not in "OGDS" or text[index + 1] != ":": + raise SddlParseError("The security descriptor is malformed.") + tag = text[index] + if tag in seen: + raise SddlParseError("The security descriptor repeats a component.") + seen.add(tag) + index += 2 + if tag in "OG": + match = _COMPONENT_PATTERN.search(text, index) + end = match.start() if match else len(text) + sid = _resolve_sid_token(text[index:end]) + if tag == "O": + owner_sid = sid + else: + group_sid = sid + index = end + continue + + flags_end = index + while flags_end < len(text) and text[flags_end] != "(" and not _COMPONENT_PATTERN.match(text, flags_end): + flags_end += 1 + flags, null_acl = _parse_acl_flags(text[index:flags_end]) + ace_texts, index = _split_aces(text, flags_end) + if index < len(text) and not _COMPONENT_PATTERN.match(text, index): + raise SddlParseError("The security descriptor is malformed.") + if tag == "D": + dacl_present = True + dacl_null = null_acl + dacl_flags = flags + dacl = tuple(_parse_ace(ace_text) for ace_text in ace_texts) + if dacl_null and dacl: + raise SddlParseError("A null DACL cannot contain ACEs.") + + return SecurityDescriptor( + owner_sid=owner_sid, + group_sid=group_sid, + dacl_present=dacl_present, + dacl_null=dacl_null, + dacl_flags=dacl_flags, + dacl=dacl, + ) + + +def _ace_membership(ace: AccessControlEntry, descriptor: SecurityDescriptor, membership: Callable[[str], str]) -> str: + if ace.sid in (CREATOR_OWNER_SID, CREATOR_GROUP_SID): + return NOT_MEMBER + if ace.sid == OWNER_RIGHTS_SID: + if not descriptor.owner_sid or descriptor.owner_sid == OWNER_RIGHTS_SID: + return UNKNOWN + return _checked_membership(descriptor.owner_sid, membership) + return _checked_membership(ace.sid, membership) + + +def _checked_membership(sid: str, membership: Callable[[str], str]) -> str: + try: + state = membership(sid) + except Exception: + return UNKNOWN + return state if state in (MEMBER, NOT_MEMBER, UNKNOWN) else UNKNOWN + + +def _uncertain_decision(unresolved: Iterable[str], unsupported: Iterable[str]) -> AccessDecision: + unresolved_sids = tuple(dict.fromkeys(unresolved)) + unsupported_aces = tuple(dict.fromkeys(unsupported)) + return AccessDecision( + outcome=OUTCOME_UNVERIFIED, + reason=REASON_SID_UNRESOLVED if unresolved_sids else REASON_ACL_UNSUPPORTED, + unresolved_sids=unresolved_sids, + unsupported_aces=unsupported_aces, + ) + + +def evaluate_read_access(descriptor: SecurityDescriptor, membership: Callable[[str], str]) -> AccessDecision: + """Decide whether a principal can read a file's data under its DACL. + + Windows evaluates a DACL in order: the first applicable ACE that names FILE_READ_DATA + decides. An ACE is uncertain when membership of its principal is unknown, when its type + is conditional or object-specific, or when it names read only through generic bits. The + outcome is definite only when every possible resolution of the uncertain ACEs before the + deciding ACE agrees with it. + """ + if not descriptor.dacl_present: + return AccessDecision(OUTCOME_UNVERIFIED, REASON_ACL_MISSING) + if descriptor.dacl_null: + return AccessDecision(OUTCOME_ALLOWED, REASON_ACL_NULL_DACL) + if not descriptor.dacl: + return AccessDecision(OUTCOME_DENIED, REASON_ACL_EMPTY) + + pending_polarities = [] + unresolved = [] + unsupported = [] + for ace in descriptor.dacl: + polarity = ACE_POLARITY.get(ace.ace_type) + if polarity is None or "IO" in ace.flags: + continue + names_read = bool(ace.mask & FILE_READ_DATA) + names_read_generically = bool(ace.mask & GENERIC_RIGHTS_INCLUDING_READ) + if not names_read and not names_read_generically: + continue + state = _ace_membership(ace, descriptor, membership) + if state == NOT_MEMBER: + continue + if state == MEMBER and ace.ace_type in SUPPORTED_ACE_TYPES and names_read: + if any(pending != polarity for pending in pending_polarities): + return _uncertain_decision(unresolved, unsupported) + if polarity == "allow": + return AccessDecision(OUTCOME_ALLOWED, REASON_ACL_ALLOW) + return AccessDecision(OUTCOME_DENIED, REASON_ACL_EXPLICIT_DENY) + pending_polarities.append(polarity) + if state == UNKNOWN: + unresolved.append(ace.sid) + if ace.ace_type not in SUPPORTED_ACE_TYPES: + unsupported.append(ace.ace_type) + elif not names_read: + unsupported.append("generic_rights") + + if "allow" in pending_polarities: + return _uncertain_decision(unresolved, unsupported) + return AccessDecision(OUTCOME_DENIED, REASON_ACL_NO_ALLOW) + + +def evaluate_sddl_read_access(sddl: str, membership: Callable[[str], str]) -> AccessDecision: + """Parse and evaluate an SDDL string; a descriptor that cannot be parsed is unverified.""" + try: + descriptor = parse_sddl(sddl) + except SddlParseError: + return AccessDecision(OUTCOME_UNVERIFIED, REASON_ACL_PARSE_ERROR) + return evaluate_read_access(descriptor, membership) + + +def build_membership( + principal_sids: Iterable[str], + *, + treat_builtin_users_as_member: bool = False, + resolve_unknown_sid: Optional[Callable[[str], str]] = None, +) -> Callable[[str], str]: + """Build a membership function for one user from the SIDs they and their groups carry. + + ``resolve_unknown_sid`` may confirm that another principal is not the user (for example + a directory lookup that finds a different user, or a group whose membership is fully + known); it can never make the user a member of anything. + """ + principals = {sid for sid in (normalize_sid(value) for value in principal_sids) if sid} + user_domains = {domain for domain, _ in (domain_sid_and_rid(sid) for sid in principals) if domain} + + def membership(sid: str) -> str: + if sid in principals or sid in ALWAYS_MEMBER_SIDS: + return MEMBER + if sid == BUILTIN_USERS_SID: + return MEMBER if treat_builtin_users_as_member else UNKNOWN + if sid in NEVER_MEMBER_SIDS: + return NOT_MEMBER + if sid.startswith("alias:"): + return NOT_MEMBER if sid[len("alias:"):] in PRIVILEGED_RELATIVE_SID_ALIASES else UNKNOWN + domain, rid = domain_sid_and_rid(sid) + if rid in PRIVILEGED_DOMAIN_RIDS: + return NOT_MEMBER + if rid == DOMAIN_USERS_RID: + return MEMBER if domain in user_domains else UNKNOWN + if resolve_unknown_sid is not None: + resolved = _checked_membership(sid, resolve_unknown_sid) + return NOT_MEMBER if resolved == NOT_MEMBER else UNKNOWN + return UNKNOWN + + return membership diff --git a/application/single_app/functions_azure_files_search.py b/application/single_app/functions_azure_files_search.py new file mode 100644 index 000000000..42f0b5631 --- /dev/null +++ b/application/single_app/functions_azure_files_search.py @@ -0,0 +1,737 @@ +# functions_azure_files_search.py +"""Search an existing Azure AI Search index built from Azure Files, returning only files the user can open. + +Customers often index Azure file shares with the Azure AI Search Azure Files indexer. The index +carries file content and paths but no permissions, and Azure AI Search cannot ingest Azure Files +ACLs. The Azure Files Search action therefore searches the customer's index, then, for every +candidate file, reads the file's security descriptor from Azure Files and evaluates it, together +with share-level permissions, against the signed-in user's directory identity. Only files the +user is proven to be able to open reach the model. Files that are denied or cannot be verified +are withheld silently and recorded for administrators. + +The functions here take their I/O dependencies as arguments so the permission logic can be +tested without Azure; ``semantic_kernel_plugins.azure_files_index_plugin`` wires the real ones. +""" + +import base64 +import logging +import re +import threading +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor, wait +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from urllib.parse import unquote, urlparse + +from functions_appinsights import log_event +from functions_azure_endpoint_validation import ( + AZURE_FILE_ENDPOINT_ERROR, + azure_storage_endpoint_suffix_for_hostname, + validate_azure_search_endpoint, +) +from functions_azure_files_access import ( + StorageAccountReference, + is_valid_share_name, + parse_storage_account_resource_id, +) +from functions_azure_files_acl import ( + MEMBER, + NOT_MEMBER, + OUTCOME_ALLOWED, + OUTCOME_DENIED, + OUTCOME_UNVERIFIED, + AccessDecision, +) + +AZURE_FILES_INDEX_ACTION_TYPE = "azure_files_index" +AZURE_FILES_SEARCH_DISPLAY_NAME = "Azure Files Search" +AZURE_FILES_SEARCH_ACTIVITY_TYPE = "azure_files_search_access" +AZURE_FILES_SEARCH_UNVERIFIED_NOTIFICATION_TYPE = "azure_files_search_access_unverified" +INTERNAL_SEARCH_INDEX_NAMES = frozenset({ + "simplechat-user-index", + "simplechat-group-index", + "simplechat-public-index", +}) + +LAYOUT_DOCUMENT_PER_FILE = "document_per_file" +LAYOUT_CHUNKED = "chunked" +LAYOUT_CUSTOM = "custom" +INDEX_LAYOUTS = (LAYOUT_DOCUMENT_PER_FILE, LAYOUT_CHUNKED, LAYOUT_CUSTOM) +LAYOUT_FIELD_DEFAULTS = { + LAYOUT_DOCUMENT_PER_FILE: { + "content_field": "content", + "title_field": "metadata_storage_name", + "path_field": "metadata_storage_path", + "name_field": "metadata_storage_name", + "last_modified_field": "metadata_storage_last_modified", + "vector_field": "", + "select_content": False, + }, + LAYOUT_CHUNKED: { + "content_field": "chunk", + "title_field": "title", + "path_field": "metadata_storage_path", + "name_field": "metadata_storage_name", + "last_modified_field": "", + "vector_field": "text_vector", + "select_content": True, + }, +} +LAYOUT_FIELD_DEFAULTS[LAYOUT_CUSTOM] = dict(LAYOUT_FIELD_DEFAULTS[LAYOUT_DOCUMENT_PER_FILE]) + +QUERY_MODE_KEYWORD = "keyword" +QUERY_MODE_SEMANTIC = "semantic" +QUERY_MODE_HYBRID = "hybrid" +QUERY_MODES = (QUERY_MODE_KEYWORD, QUERY_MODE_SEMANTIC, QUERY_MODE_HYBRID) + +PERMISSION_MODE_LIVE_ACL = "live_acl" +PERMISSION_MODE_NONE = "none" +PERMISSION_MODES = (PERMISSION_MODE_LIVE_ACL, PERMISSION_MODE_NONE) +SHARE_ACCESS_CHECK_RBAC = "rbac" +SHARE_ACCESS_CHECK_SKIP = "skip" +SHARE_ACCESS_CHECKS = (SHARE_ACCESS_CHECK_RBAC, SHARE_ACCESS_CHECK_SKIP) + +DEFAULT_TOP_N = 5 +MAX_TOP_N = 20 +DEFAULT_MAX_CANDIDATES = 50 +MAX_CANDIDATES = 200 +MIN_CANDIDATES = 5 +DEFAULT_MAX_SNIPPET_CHARS = 1200 +MIN_SNIPPET_CHARS = 200 +MAX_SNIPPET_CHARS = 4000 +DEFAULT_TIME_BUDGET_SECONDS = 25 +MIN_TIME_BUDGET_SECONDS = 5 +MAX_TIME_BUDGET_SECONDS = 60 +MAX_RESULTS_PER_FILE = 3 +MAX_STORAGE_SHARES = 50 +MAX_LOGGED_WITHHELD_FILES = 25 +MAX_LOGGED_UNRESOLVED_SIDS = 5 +MAX_QUERY_LENGTH = 1000 +ACL_EVALUATION_WORKERS = 8 + +REASON_PATH_MISSING = "path_missing" +REASON_STORAGE_NOT_ALLOWLISTED = "storage_not_allowlisted" +REASON_FILE_NOT_FOUND = "file_not_found" +REASON_ACL_READ_FAILED = "acl_read_failed" +REASON_TIME_BUDGET_EXCEEDED = "time_budget_exceeded" +REASON_PERMISSION_CHECKS_DISABLED = "permission_checks_disabled" + +_INDEX_NAME_PATTERN = re.compile(r"^[a-z0-9](?:[a-z0-9]|-(?=[a-z0-9])){1,127}$") +_FIELD_NAME_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,127}$") +_SEMANTIC_CONFIGURATION_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +_WHITESPACE_PATTERN = re.compile(r"\s+") + + +class AzureFilesSearchConfigError(ValueError): + """An Azure Files Search configuration is invalid; the message is safe to show admins.""" + + def __init__(self, public_message: str): + self.public_message = public_message + super().__init__(public_message) + + +class AzureFileNotFound(RuntimeError): + """The indexed file no longer exists in the share.""" + + +@dataclass(frozen=True) +class StorageShareEntry: + account: StorageAccountReference + share_name: str + + +@dataclass(frozen=True) +class AzureFilesSearchConfig: + action_id: str + action_name: str + display_name: str + endpoint: str + index_name: str + auth_type: str + auth_key: str = field(repr=False) + layout: str + content_field: str + title_field: str + path_field: str + name_field: str + last_modified_field: str + vector_field: str + select_content: bool + query_mode: str + semantic_configuration: str + default_top_n: int + max_candidates: int + max_snippet_chars: int + time_budget_seconds: int + permission_mode: str + share_access_check: str + treat_builtin_users_as_member: bool + storage_shares: Tuple[StorageShareEntry, ...] + + def find_share(self, account_name: str, share_name: str) -> Optional[StorageShareEntry]: + for entry in self.storage_shares: + if entry.account.account_name == account_name and entry.share_name == share_name: + return entry + return None + + +@dataclass(frozen=True) +class AzureFilePath: + url: str + file_endpoint: str + account_name: str + endpoint_suffix: str + share_name: str + relative_path: str + file_name: str + unc_path: str + folder: str + + +@dataclass +class AzureFilesSearchOutcome: + results: List[Dict[str, Any]] + candidate_count: int = 0 + files_evaluated: int = 0 + allowed_files: int = 0 + denied_files: int = 0 + unverified_files: int = 0 + reasons: Dict[str, int] = field(default_factory=dict) + withheld_files: List[Dict[str, Any]] = field(default_factory=list) + withheld_files_truncated: bool = False + duration_ms: int = 0 + error: str = "" + + @property + def withheld_count(self) -> int: + return self.denied_files + self.unverified_files + + +def _text(value: Any, limit: int = 256) -> str: + return str(value or "").strip()[:limit] + + +def _as_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + return str(value or "").strip().lower() in {"1", "true", "yes", "on"} + + +def _bounded_int(value: Any, default: int, minimum: int, maximum: int) -> int: + try: + number = int(value) + except (TypeError, ValueError): + return default + return max(minimum, min(maximum, number)) + + +def _field_name(value: Any, default: str, label: str, required: bool = False) -> str: + name = _text(value, 128) if value is not None else default + if not name: + if required: + raise AzureFilesSearchConfigError(f"Enter the {label} field name.") + return "" + if not _FIELD_NAME_PATTERN.match(name): + raise AzureFilesSearchConfigError(f"The {label} field name is invalid.") + return name + + +def normalize_azure_files_search_config(manifest: Dict[str, Any]) -> AzureFilesSearchConfig: + """Validate an Azure Files Search action manifest and return its normalized configuration.""" + if not isinstance(manifest, dict): + raise AzureFilesSearchConfigError("The Azure Files Search configuration is invalid.") + fields = manifest.get("additionalFields") if isinstance(manifest.get("additionalFields"), dict) else {} + auth = manifest.get("auth") if isinstance(manifest.get("auth"), dict) else {} + + try: + endpoint = validate_azure_search_endpoint(manifest.get("endpoint")) + except ValueError as error: + raise AzureFilesSearchConfigError(str(error)) from error + + index_name = _text(fields.get("index_name"), 128).lower() + if not _INDEX_NAME_PATTERN.match(index_name): + raise AzureFilesSearchConfigError( + "Enter the search index name: 2-128 lowercase letters, numbers, or single dashes." + ) + + auth_type = _text(auth.get("type") or "identity", 32).lower() + if auth_type not in {"identity", "key"}: + raise AzureFilesSearchConfigError("Azure Files Search supports managed identity or a query key.") + auth_key = str(auth.get("key") or "").strip() if auth_type == "key" else "" + if auth_type == "key" and not auth_key: + raise AzureFilesSearchConfigError("Enter the search query key, or use managed identity.") + + layout = _text(fields.get("index_layout") or LAYOUT_DOCUMENT_PER_FILE, 32) + if layout not in INDEX_LAYOUTS: + raise AzureFilesSearchConfigError("Choose a supported index layout.") + defaults = LAYOUT_FIELD_DEFAULTS[layout] + use_defaults = layout != LAYOUT_CUSTOM + + def configured(name: str) -> Any: + return None if use_defaults and not fields.get(name) else fields.get(name, defaults[name]) + + content_field = _field_name(configured("content_field"), defaults["content_field"], "content", required=True) + path_field = _field_name(configured("path_field"), defaults["path_field"], "file path", required=True) + name_field = _field_name(configured("name_field"), defaults["name_field"], "file name", required=True) + title_field = _field_name(configured("title_field"), defaults["title_field"], "title") + last_modified_field = _field_name(configured("last_modified_field"), defaults["last_modified_field"], "last modified") + vector_field = _field_name(configured("vector_field"), defaults["vector_field"], "vector") + select_content = ( + _as_bool(fields["select_content"]) if "select_content" in fields else defaults["select_content"] + ) + + query_mode = _text(fields.get("query_mode") or QUERY_MODE_KEYWORD, 32) + if query_mode not in QUERY_MODES: + raise AzureFilesSearchConfigError("Choose keyword, semantic, or hybrid search.") + semantic_configuration = _text(fields.get("semantic_configuration"), 128) + if semantic_configuration and not _SEMANTIC_CONFIGURATION_PATTERN.match(semantic_configuration): + raise AzureFilesSearchConfigError("The semantic configuration name is invalid.") + if query_mode == QUERY_MODE_HYBRID and not vector_field: + raise AzureFilesSearchConfigError("Hybrid search needs the index's vector field.") + + permission_mode = _text(fields.get("permission_mode") or PERMISSION_MODE_LIVE_ACL, 32) + if permission_mode not in PERMISSION_MODES: + raise AzureFilesSearchConfigError("Choose how file permissions are checked.") + if permission_mode == PERMISSION_MODE_NONE and not _as_bool(fields.get("permission_mode_none_acknowledged")): + raise AzureFilesSearchConfigError( + "Turning off file permission checks lets everyone who can use this action search every file in the index. " + "Confirm the acknowledgment to continue." + ) + share_access_check = _text(fields.get("share_access_check") or SHARE_ACCESS_CHECK_RBAC, 32) + if share_access_check not in SHARE_ACCESS_CHECKS: + raise AzureFilesSearchConfigError("Choose how share-level permissions are checked.") + + storage_shares = [] + raw_shares = fields.get("storage_shares") if isinstance(fields.get("storage_shares"), list) else [] + if len(raw_shares) > MAX_STORAGE_SHARES: + raise AzureFilesSearchConfigError(f"List at most {MAX_STORAGE_SHARES} file shares.") + seen = set() + for raw_share in raw_shares: + raw_share = raw_share if isinstance(raw_share, dict) else {} + try: + account = parse_storage_account_resource_id(raw_share.get("storage_account_resource_id")) + except ValueError as error: + raise AzureFilesSearchConfigError(str(error)) from error + share_name = _text(raw_share.get("share_name"), 63).lower() + if not is_valid_share_name(share_name): + raise AzureFilesSearchConfigError("Each file share needs a valid share name.") + key = (account.account_name, share_name) + if key in seen: + continue + seen.add(key) + storage_shares.append(StorageShareEntry(account=account, share_name=share_name)) + if permission_mode == PERMISSION_MODE_LIVE_ACL and not storage_shares: + raise AzureFilesSearchConfigError("Add the storage account and file share that this index was built from.") + + return AzureFilesSearchConfig( + action_id=_text(manifest.get("id"), 128), + action_name=_text(manifest.get("name"), 128), + display_name=_text(manifest.get("displayName") or manifest.get("name") or AZURE_FILES_SEARCH_DISPLAY_NAME, 128), + endpoint=endpoint, + index_name=index_name, + auth_type=auth_type, + auth_key=auth_key, + layout=layout, + content_field=content_field, + title_field=title_field, + path_field=path_field, + name_field=name_field, + last_modified_field=last_modified_field, + vector_field=vector_field, + select_content=select_content, + query_mode=query_mode, + semantic_configuration=semantic_configuration, + default_top_n=_bounded_int(fields.get("default_top_n"), DEFAULT_TOP_N, 1, MAX_TOP_N), + max_candidates=_bounded_int(fields.get("max_candidates"), DEFAULT_MAX_CANDIDATES, MIN_CANDIDATES, MAX_CANDIDATES), + max_snippet_chars=_bounded_int( + fields.get("max_snippet_chars"), DEFAULT_MAX_SNIPPET_CHARS, MIN_SNIPPET_CHARS, MAX_SNIPPET_CHARS, + ), + time_budget_seconds=_bounded_int( + fields.get("time_budget_seconds"), DEFAULT_TIME_BUDGET_SECONDS, MIN_TIME_BUDGET_SECONDS, MAX_TIME_BUDGET_SECONDS, + ), + permission_mode=permission_mode, + share_access_check=share_access_check, + treat_builtin_users_as_member=_as_bool(fields.get("treat_builtin_users_as_member")), + storage_shares=tuple(storage_shares), + ) + + +def internal_search_endpoints(settings: Dict[str, Any]) -> Tuple[str, ...]: + """Return SimpleChat's own Azure AI Search service origins from settings.""" + endpoints = [] + for key in ("azure_ai_search_endpoint", "target_ai_search_endpoint"): + try: + endpoints.append(validate_azure_search_endpoint((settings or {}).get(key))) + except ValueError: + continue + return tuple(dict.fromkeys(endpoints)) + + +def assert_not_internal_index( + config: AzureFilesSearchConfig, + settings: Dict[str, Any], + alias_exists: Optional[Callable[[str], Optional[bool]]] = None, +) -> None: + """Refuse SimpleChat's own indexes, which hold every workspace's documents. + + On SimpleChat's own search service an index alias could point at an internal index, so + aliases are refused there too. When the alias lookup fails the index is refused, because + it cannot be proven safe. + """ + if config.endpoint not in internal_search_endpoints(settings): + return + if config.index_name in INTERNAL_SEARCH_INDEX_NAMES: + raise AzureFilesSearchConfigError("SimpleChat's own workspace indexes can't be used with Azure Files Search.") + if alias_exists is None: + return + exists = alias_exists(config.index_name) + if exists is None or exists: + raise AzureFilesSearchConfigError( + "Index aliases on SimpleChat's own search service can't be used with Azure Files Search. Use the index name." + ) + + +def _decode_path_value(value: Any) -> str: + text = str(value or "").strip() + if text.lower().startswith("https://"): + return text + # Indexers often store the document key as base64 of the path, with a trailing padding digit. + candidate = text[:-1] if text[-1:].isdigit() else text + try: + padded = candidate + "=" * (-len(candidate) % 4) + decoded = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8") + except (ValueError, UnicodeError): + return "" + return decoded if decoded.lower().startswith("https://") else "" + + +def parse_azure_file_path(value: Any) -> Optional[AzureFilePath]: + """Parse an indexed Azure Files URL into its account, share, and file path.""" + url = _decode_path_value(value) + if not url: + return None + parsed = urlparse(url) + if parsed.scheme != "https" or not parsed.hostname or parsed.query or parsed.fragment or parsed.port not in (None, 443): + return None + try: + account_name, endpoint_suffix = azure_storage_endpoint_suffix_for_hostname( + parsed.hostname.lower(), "file", AZURE_FILE_ENDPOINT_ERROR, + ) + except ValueError: + return None + segments = [unquote(segment) for segment in parsed.path.split("/") if segment] + if len(segments) < 2: + return None + share_name = segments[0].lower() + file_segments = segments[1:] + if not is_valid_share_name(share_name) or any( + segment in {".", ".."} or "\\" in segment or "\x00" in segment for segment in file_segments + ): + return None + host = f"{account_name}.file.{endpoint_suffix}" + return AzureFilePath( + url=url, + file_endpoint=f"https://{host}", + account_name=account_name, + endpoint_suffix=endpoint_suffix, + share_name=share_name, + relative_path="/".join(file_segments), + file_name=file_segments[-1], + unc_path="\\\\" + "\\".join([host, share_name, *file_segments]), + folder="\\".join(file_segments[:-1]), + ) + + +def _clip(text: Any, limit: int) -> str: + normalized = _WHITESPACE_PATTERN.sub(" ", str(text or "")).strip() + if len(normalized) <= limit: + return normalized + return normalized[: max(0, limit - 1)].rstrip() + "\u2026" + + +def _caption_texts(hit: Dict[str, Any]) -> List[str]: + texts = [] + for caption in hit.get("@search.captions") or []: + text = getattr(caption, "text", None) + if text is None and isinstance(caption, dict): + text = caption.get("text") + if text: + texts.append(str(text)) + return texts + + +def build_snippet(hit: Dict[str, Any], config: AzureFilesSearchConfig) -> str: + """Prefer semantic captions, then highlights, then the selected content, clipped.""" + captions = _caption_texts(hit) + if captions: + return _clip(" ".join(captions), config.max_snippet_chars) + highlights = (hit.get("@search.highlights") or {}).get(config.content_field) or [] + if highlights: + return _clip(" \u2026 ".join(str(item) for item in highlights), config.max_snippet_chars) + if config.select_content: + return _clip(hit.get(config.content_field), config.max_snippet_chars) + return "" + + +def build_search_request(config: AzureFilesSearchConfig, query: str, vector_query_factory: Optional[Callable[..., Any]]) -> Dict[str, Any]: + """Build keyword arguments for ``SearchClient.search``; the model controls only the query text.""" + select = [config.path_field, config.name_field] + for optional_field in (config.title_field, config.last_modified_field): + if optional_field and optional_field not in select: + select.append(optional_field) + if config.select_content and config.content_field not in select: + select.append(config.content_field) + request: Dict[str, Any] = { + "search_text": query, + "select": select, + "top": config.max_candidates, + } + if not config.select_content: + request["highlight_fields"] = config.content_field + request["highlight_pre_tag"] = "" + request["highlight_post_tag"] = "" + if config.query_mode in (QUERY_MODE_SEMANTIC, QUERY_MODE_HYBRID) and ( + config.query_mode == QUERY_MODE_SEMANTIC or config.semantic_configuration + ): + request["query_type"] = "semantic" + if config.semantic_configuration: + request["semantic_configuration_name"] = config.semantic_configuration + request["query_caption"] = "extractive" + if config.query_mode == QUERY_MODE_HYBRID and vector_query_factory is not None: + request["vector_queries"] = [ + vector_query_factory(text=query, k_nearest_neighbors=config.max_candidates, fields=config.vector_field) + ] + return request + + +@dataclass(frozen=True) +class SearchDependencies: + """I/O used by one search: search hits, file security descriptors, share access, identity.""" + + run_search: Callable[[Dict[str, Any]], Sequence[Dict[str, Any]]] + read_file_sddl: Callable[[AzureFilePath], str] + check_share_access: Callable[[StorageShareEntry], Tuple[str, str]] + membership: Optional[Callable[[str], str]] + evaluate_sddl: Callable[[str, Callable[[str], str]], AccessDecision] + identity_error_reason: str = "" + + +def _withheld_entry(path: Optional[AzureFilePath], raw_path: str, decision: AccessDecision) -> Dict[str, Any]: + entry = { + "file": path.unc_path if path else _text(raw_path, 512), + "outcome": decision.outcome, + "reason": decision.reason, + } + if decision.unresolved_sids: + entry["unresolved_sids"] = list(decision.unresolved_sids[:MAX_LOGGED_UNRESOLVED_SIDS]) + if decision.unsupported_aces: + entry["unsupported"] = list(decision.unsupported_aces[:MAX_LOGGED_UNRESOLVED_SIDS]) + return entry + + +def _decide_file(config: AzureFilesSearchConfig, path: Optional[AzureFilePath], dependencies: SearchDependencies) -> AccessDecision: + if config.permission_mode == PERMISSION_MODE_NONE: + return AccessDecision(OUTCOME_ALLOWED, REASON_PERMISSION_CHECKS_DISABLED) + if path is None: + return AccessDecision(OUTCOME_UNVERIFIED, REASON_PATH_MISSING) + share = config.find_share(path.account_name, path.share_name) + if share is None: + return AccessDecision(OUTCOME_UNVERIFIED, REASON_STORAGE_NOT_ALLOWLISTED) + if dependencies.membership is None: + return AccessDecision(OUTCOME_UNVERIFIED, dependencies.identity_error_reason or "identity_unavailable") + if config.share_access_check == SHARE_ACCESS_CHECK_RBAC: + share_state, share_reason = dependencies.check_share_access(share) + if share_state == NOT_MEMBER: + return AccessDecision(OUTCOME_DENIED, share_reason) + if share_state != MEMBER: + return AccessDecision(OUTCOME_UNVERIFIED, share_reason) + try: + sddl = dependencies.read_file_sddl(path) + except AzureFileNotFound: + return AccessDecision(OUTCOME_UNVERIFIED, REASON_FILE_NOT_FOUND) + except Exception as error: + log_event( + "[AZURE_FILES_SEARCH] Could not read a file's permissions.", + level=logging.WARNING, + extra={ + "exception_type": type(error).__name__, + "storage_account": path.account_name, + "share": path.share_name, + "action_id": config.action_id, + }, + ) + return AccessDecision(OUTCOME_UNVERIFIED, REASON_ACL_READ_FAILED) + return dependencies.evaluate_sddl(sddl, dependencies.membership) + + +def _hit_score(hit: Dict[str, Any]) -> Optional[float]: + for key in ("@search.reranker_score", "@search.rerankerScore", "@search.score"): + value = hit.get(key) + if isinstance(value, (int, float)): + return round(float(value), 4) + return None + + +def run_azure_files_search( + config: AzureFilesSearchConfig, + query: str, + top_n: Optional[int], + dependencies: SearchDependencies, + vector_query_factory: Optional[Callable[..., Any]] = None, + clock: Callable[[], float] = time.monotonic, +) -> AzureFilesSearchOutcome: + """Search the index and keep only results whose files the user is proven able to open.""" + started_at = clock() + deadline = started_at + config.time_budget_seconds + limit = _bounded_int(top_n, config.default_top_n, 1, MAX_TOP_N) if top_n else config.default_top_n + query_text = _text(query, MAX_QUERY_LENGTH) + outcome = AzureFilesSearchOutcome(results=[]) + if not query_text: + outcome.error = "Enter what to search for." + return outcome + + hits = list(dependencies.run_search(build_search_request(config, query_text, vector_query_factory)) or []) + outcome.candidate_count = len(hits) + + file_order: List[str] = [] + paths: Dict[str, Optional[AzureFilePath]] = {} + for hit in hits: + raw_path = str(hit.get(config.path_field) or "") + key = raw_path or f"missing:{len(file_order)}" + if key not in paths: + paths[key] = parse_azure_file_path(raw_path) + file_order.append(key) + hit["_simplechat_file_key"] = key + + decisions: Dict[str, AccessDecision] = {} + executor = ThreadPoolExecutor(max_workers=ACL_EVALUATION_WORKERS, thread_name_prefix="azure-files-acl") + try: + futures = {executor.submit(_decide_file, config, paths[key], dependencies): key for key in file_order} + done, not_done = wait(futures, timeout=max(0.0, deadline - clock())) + for future in done: + key = futures[future] + try: + decisions[key] = future.result() + except Exception: + decisions[key] = AccessDecision(OUTCOME_UNVERIFIED, REASON_ACL_READ_FAILED) + for future in not_done: + future.cancel() + decisions[futures[future]] = AccessDecision(OUTCOME_UNVERIFIED, REASON_TIME_BUDGET_EXCEEDED) + finally: + executor.shutdown(wait=False, cancel_futures=True) + + reasons: Counter = Counter() + for key in file_order: + decision = decisions[key] + outcome.files_evaluated += 1 + if decision.outcome == OUTCOME_ALLOWED: + outcome.allowed_files += 1 + continue + reasons[decision.reason] += 1 + if decision.outcome == OUTCOME_DENIED: + outcome.denied_files += 1 + else: + outcome.unverified_files += 1 + if len(outcome.withheld_files) < MAX_LOGGED_WITHHELD_FILES: + outcome.withheld_files.append(_withheld_entry(paths[key], key, decision)) + else: + outcome.withheld_files_truncated = True + outcome.reasons = dict(reasons) + + per_file_counts: Counter = Counter() + for hit in hits: + key = hit["_simplechat_file_key"] + if decisions[key].outcome != OUTCOME_ALLOWED or per_file_counts[key] >= MAX_RESULTS_PER_FILE: + continue + per_file_counts[key] += 1 + path = paths[key] + result = { + "file_name": path.file_name if path else _text(hit.get(config.name_field), 256), + "unc_path": path.unc_path if path else "", + "folder": path.folder if path else "", + "share": path.share_name if path else "", + "title": _text(hit.get(config.title_field), 512) if config.title_field else "", + "last_modified": _text(hit.get(config.last_modified_field), 64) if config.last_modified_field else "", + "snippet": build_snippet(hit, config), + "score": _hit_score(hit), + } + outcome.results.append({name: value for name, value in result.items() if value not in ("", None)}) + if len(outcome.results) >= limit: + break + + outcome.duration_ms = int((clock() - started_at) * 1000) + return outcome + + +def build_model_response(config: AzureFilesSearchConfig, outcome: AzureFilesSearchOutcome) -> Dict[str, Any]: + """Return what the model sees: only allowed results, never counts of withheld files.""" + if outcome.error: + return {"results": [], "result_count": 0, "error": outcome.error} + return { + "source": config.display_name, + "results": outcome.results, + "result_count": len(outcome.results), + "citation_guidance": "Cite each file by its file name and network path (unc_path).", + } + + +def build_review_record(config: AzureFilesSearchConfig, outcome: AzureFilesSearchOutcome) -> Dict[str, Any]: + """Summarize one search for the admin-only activity log; contains no file content or query text.""" + return { + "action_id": config.action_id, + "action_name": config.action_name, + "display_name": config.display_name, + "search_service": urlparse(config.endpoint).hostname or "", + "index_name": config.index_name, + "permission_mode": config.permission_mode, + "share_access_check": config.share_access_check, + "status": "unverified" if outcome.unverified_files else ("denied" if outcome.denied_files else "allowed"), + "counts": { + "candidates": outcome.candidate_count, + "files_evaluated": outcome.files_evaluated, + "allowed_files": outcome.allowed_files, + "denied_files": outcome.denied_files, + "unverified_files": outcome.unverified_files, + "results_returned": len(outcome.results), + }, + "reasons": dict(outcome.reasons), + "withheld_files": list(outcome.withheld_files), + "withheld_files_truncated": outcome.withheld_files_truncated, + "duration_ms": outcome.duration_ms, + } + + +class SddlCache: + """Cache security descriptors by permission key; files with identical ACLs share one key.""" + + def __init__(self, ttl_seconds: int = 300, max_entries: int = 5000): + self._ttl_seconds = ttl_seconds + self._max_entries = max_entries + self._entries: Dict[Tuple[str, str, str], Tuple[float, str]] = {} + self._lock = threading.Lock() + + def get_or_load(self, key: Tuple[str, str, str], loader: Callable[[], str]) -> str: + now = time.monotonic() + with self._lock: + entry = self._entries.get(key) + if entry and entry[0] > now: + return entry[1] + value = loader() + with self._lock: + if len(self._entries) >= self._max_entries: + self._entries.clear() + self._entries[key] = (now + self._ttl_seconds, value) + return value + + +def notification_day(now: Optional[datetime] = None) -> str: + return (now or datetime.now(timezone.utc)).strftime("%Y-%m-%d") + + +def validate_azure_files_index_action(manifest: Dict[str, Any], settings: Dict[str, Any]) -> AzureFilesSearchConfig: + """Validate an Azure Files Search action for saving, refusing SimpleChat's own indexes.""" + config = normalize_azure_files_search_config(manifest) + assert_not_internal_index(config, settings) + return config diff --git a/application/single_app/functions_azure_files_search_runtime.py b/application/single_app/functions_azure_files_search_runtime.py new file mode 100644 index 000000000..09200f2ce --- /dev/null +++ b/application/single_app/functions_azure_files_search_runtime.py @@ -0,0 +1,494 @@ +# functions_azure_files_search_runtime.py +"""Connect the Azure Files Search action to Azure AI Search, Azure Files, Graph, and ARM. + +``functions_azure_files_search`` holds the search and permission logic with its I/O injected. +This module supplies the real clients, records each search for administrators, and runs the +admin connection test. The application identity reads the customer's search index (Search +Index Data Reader) and each file's security descriptor (Storage File Data Privileged Reader, +backup intent); the signed-in user's delegated Graph token resolves their group SIDs. +""" + +import logging +import threading +from typing import Any, Callable, Dict, List, Optional, Tuple + +from azure.core.credentials import AzureKeyCredential +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.identity import DefaultAzureCredential +from azure.search.documents import SearchClient +from azure.search.documents.models import VectorizableTextQuery +from azure.storage.fileshare import ShareServiceClient +from flask import g, has_app_context + +from agent_execution_context import current_agent_execution +from config import AZURE_ENVIRONMENT, resource_manager, search_resource_manager +from functions_action_manifest import get_action_origin +from functions_appinsights import log_event +from functions_authentication import get_graph_base_url, get_valid_access_token +from functions_azure_files_access import ( + AzureFilesIdentityError, + check_share_level_access, + classify_foreign_sid, + get_user_principals, +) +from functions_azure_files_acl import MEMBER, NOT_MEMBER, build_membership, evaluate_sddl_read_access +from functions_azure_files_search import ( + AZURE_FILES_SEARCH_UNVERIFIED_NOTIFICATION_TYPE, + PERMISSION_MODE_NONE, + SHARE_ACCESS_CHECK_RBAC, + AzureFilePath, + AzureFileNotFound, + AzureFilesSearchConfig, + AzureFilesSearchConfigError, + AzureFilesSearchOutcome, + SddlCache, + SearchDependencies, + StorageShareEntry, + assert_not_internal_index, + build_model_response, + build_review_record, + build_search_request, + normalize_azure_files_search_config, + notification_day, + parse_azure_file_path, + run_azure_files_search, +) + +# The GA Search API cannot address index aliases, so a configured name always means one index. +AZURE_FILES_SEARCH_API_VERSION = "2024-07-01" +AZURE_FILES_TOKEN_INTENT = "backup" +SEARCH_CONNECTION_TIMEOUT_SECONDS = 10 +SEARCH_READ_TIMEOUT_SECONDS = 30 +FILE_REQUEST_TIMEOUT_SECONDS = 10 +UNAVAILABLE_MESSAGE = "Azure Files Search is unavailable right now. Try again later." +MISCONFIGURED_MESSAGE = "This Azure Files Search action isn't configured correctly. Ask an administrator to check it." +SIGNED_OUT_MESSAGE = "Azure Files Search needs a signed-in user." + +_credential_lock = threading.Lock() +_credential: Optional[DefaultAzureCredential] = None +_share_service_clients: Dict[str, ShareServiceClient] = {} +_share_service_lock = threading.Lock() +_sddl_cache = SddlCache() + + +class AzureFilesSearchUnavailable(RuntimeError): + """A search request failed; ``public_message`` is a reviewed, actionable explanation.""" + + def __init__(self, code: str, public_message: str): + self.code = code + self.public_message = public_message + super().__init__(code) + + +def _app_credential() -> DefaultAzureCredential: + global _credential + with _credential_lock: + if _credential is None: + _credential = DefaultAzureCredential() + return _credential + + +def _search_client(config: AzureFilesSearchConfig) -> SearchClient: + options: Dict[str, Any] = { + "api_version": AZURE_FILES_SEARCH_API_VERSION, + "connection_timeout": SEARCH_CONNECTION_TIMEOUT_SECONDS, + "read_timeout": SEARCH_READ_TIMEOUT_SECONDS, + } + if config.auth_type == "key": + credential: Any = AzureKeyCredential(config.auth_key) + else: + credential = _app_credential() + if AZURE_ENVIRONMENT in ("usgovernment", "custom"): + options["audience"] = search_resource_manager + return SearchClient(endpoint=config.endpoint, index_name=config.index_name, credential=credential, **options) + + +def classify_search_error(error: Exception, config: AzureFilesSearchConfig) -> AzureFilesSearchUnavailable: + """Map an Azure AI Search failure to a reviewed message an administrator can act on.""" + status_code = getattr(error, "status_code", None) + detail = str(getattr(error, "message", "") or "").lower() + if status_code in (401, 403): + if config.auth_type == "key": + return AzureFilesSearchUnavailable( + "search_access_denied", "Azure AI Search rejected the query key. Check the key stored for this action.", + ) + return AzureFilesSearchUnavailable( + "search_access_denied", + "Azure AI Search denied the query. Give SimpleChat's managed identity the Search Index Data Reader role " + "on the search service, and enable role-based access on the service.", + ) + if status_code == 404 or isinstance(error, ResourceNotFoundError): + return AzureFilesSearchUnavailable("search_index_not_found", "The search index wasn't found on the search service.") + if status_code == 400 and "semantic" in detail: + return AzureFilesSearchUnavailable( + "search_semantic_unavailable", + "The index has no usable semantic configuration. Enter its name, or use keyword or hybrid search.", + ) + if status_code == 400 and ("vector" in detail or "vectorizer" in detail): + return AzureFilesSearchUnavailable( + "search_vector_unavailable", + "The vector field has no vectorizer, so hybrid search can't run. Use keyword or semantic search.", + ) + if status_code == 400: + return AzureFilesSearchUnavailable( + "search_request_invalid", + "Azure AI Search rejected the request. Check that the configured field names exist and are retrievable.", + ) + return AzureFilesSearchUnavailable("search_unavailable", UNAVAILABLE_MESSAGE) + + +def _run_search_for(config: AzureFilesSearchConfig) -> Callable[[Dict[str, Any]], List[Dict[str, Any]]]: + def run(request: Dict[str, Any]) -> List[Dict[str, Any]]: + try: + hits = [] + for hit in _search_client(config).search(**request): + hits.append(dict(hit)) + if len(hits) >= config.max_candidates: + break + return hits + except HttpResponseError as error: + raise classify_search_error(error, config) from error + + return run + + +def _share_service_client(file_endpoint: str) -> ShareServiceClient: + with _share_service_lock: + client = _share_service_clients.get(file_endpoint) + if client is None: + client = ShareServiceClient( + account_url=file_endpoint, + credential=_app_credential(), + token_intent=AZURE_FILES_TOKEN_INTENT, + connection_timeout=FILE_REQUEST_TIMEOUT_SECONDS, + read_timeout=FILE_REQUEST_TIMEOUT_SECONDS, + ) + _share_service_clients[file_endpoint] = client + return client + + +def read_file_sddl(path: AzureFilePath) -> str: + """Read a file's security descriptor; files with identical ACLs share one cached lookup.""" + share_client = _share_service_client(path.file_endpoint).get_share_client(path.share_name) + try: + properties = share_client.get_file_client(path.relative_path).get_file_properties( + timeout=FILE_REQUEST_TIMEOUT_SECONDS, + ) + except ResourceNotFoundError as error: + raise AzureFileNotFound("file_not_found") from error + permission_key = str(getattr(properties, "permission_key", "") or "") + if not permission_key: + raise RuntimeError("permission_key_missing") + return _sddl_cache.get_or_load( + (path.account_name, path.share_name, permission_key), + lambda: share_client.get_permission_for_share(permission_key, timeout=FILE_REQUEST_TIMEOUT_SECONDS), + ) + + +def _arm_token() -> str: + return _app_credential().get_token(f"{resource_manager.rstrip('/')}/.default").token + + +def _check_share_access_for(user_id: str) -> Callable[[StorageShareEntry], Tuple[str, str]]: + def check(share: StorageShareEntry) -> Tuple[str, str]: + return check_share_level_access(user_id, share.account, share.share_name, resource_manager, _arm_token) + + return check + + +def build_user_membership(config: AzureFilesSearchConfig, user_id: str) -> Tuple[Optional[Callable[[str], str]], str]: + """Resolve the signed-in user's SIDs; ``(None, reason)`` when they cannot be resolved.""" + access_token = get_valid_access_token() + if not access_token: + return None, "identity_unavailable" + graph_base_url = get_graph_base_url() + try: + principals = get_user_principals(user_id, access_token, graph_base_url) + except AzureFilesIdentityError as error: + return None, error.reason + membership = build_membership( + principals.sids, + treat_builtin_users_as_member=config.treat_builtin_users_as_member, + resolve_unknown_sid=lambda sid: classify_foreign_sid(sid, access_token, graph_base_url), + ) + return membership, "" + + +def _context_text(value: Any) -> str: + return value.strip()[:200] if isinstance(value, str) else "" + + +def _invocation_context() -> Dict[str, Any]: + """Return the conversation and agent that ran the search, for the admin review only. + + Chat, workflow, and delegated agent runs set these on ``g``; a delegated or orchestrated run + also carries the conversation on its execution frame. Only identifiers and names are copied, + never the agent's instructions or any other configuration. + """ + frame = current_agent_execution() + conversation_id = frame.identity.conversation_id if frame is not None else None + agent_info, agent_name = None, None + if has_app_context(): + conversation_id = conversation_id or getattr(g, "conversation_id", None) + agent_info = getattr(g, "request_agent_info", None) + agent_name = getattr(g, "request_agent_name", None) + agent_info = agent_info if isinstance(agent_info, dict) else {} + name = _context_text(agent_info.get("name")) or _context_text(agent_name) + agent = { + "id": _context_text(agent_info.get("id")), + "name": name, + "display_name": _context_text(agent_info.get("display_name")) or name, + } + return { + "conversation_id": _context_text(conversation_id), + "agent": agent if agent["id"] or agent["name"] else None, + } + + +def record_azure_files_search_review( + config: AzureFilesSearchConfig, + user_id: str, + outcome: AzureFilesSearchOutcome, +) -> None: + """Record a search for administrators: always in telemetry, in activity logs when files were withheld.""" + review = {**build_review_record(config, outcome), **_invocation_context()} + log_event( + "[AZURE_FILES_SEARCH] Search completed.", + extra={"user_id": user_id, **{key: value for key, value in review.items() if key != "withheld_files"}}, + level=logging.INFO, + ) + if not outcome.withheld_count: + return + try: + # Imported at use: plugin discovery imports this module, and activity logging and + # notifications pull in workspace and workflow modules that discovery doesn't need. + from functions_activity_logging import log_azure_files_search_access + + log_azure_files_search_access(user_id=user_id, review=review) + except Exception as error: + log_event( + "[AZURE_FILES_SEARCH] Failed to record the access review.", + level=logging.WARNING, + extra={"exception_type": type(error).__name__, "action_id": config.action_id}, + ) + if outcome.unverified_files: + _notify_admins_of_unverified_results(config, review) + + +def _notify_admins_of_unverified_results(config: AzureFilesSearchConfig, review: Dict[str, Any]) -> None: + try: + # Imported at use for the same reason as the activity-logging hook above. + from functions_notifications import create_notification + + create_notification( + notification_type=AZURE_FILES_SEARCH_UNVERIFIED_NOTIFICATION_TYPE, + title=f"Azure Files Search could not verify file permissions: {config.display_name}", + message=( + "Some results were withheld because their file permissions could not be verified. " + "Review the Azure Files Search access entries in the Control Center activity logs." + ), + link_url="/admin/control-center", + link_context={"activity_type": "azure_files_search_access", "action_id": config.action_id}, + metadata={ + "action_id": config.action_id, + "index_name": config.index_name, + "reasons": review.get("reasons", {}), + }, + assignment={"roles": ["Admin"]}, + idempotency_key=f"azure-files-search-unverified:{config.action_id or config.index_name}:{notification_day()}", + ) + except Exception as error: + log_event( + "[AZURE_FILES_SEARCH] Failed to notify administrators about unverified results.", + level=logging.WARNING, + extra={"exception_type": type(error).__name__, "action_id": config.action_id}, + ) + + +def execute_azure_files_search( + manifest: Dict[str, Any], + query: str, + top_n: Optional[int], + user_id: str, + settings: Dict[str, Any], +) -> Dict[str, Any]: + """Run one Azure Files Search for the signed-in user and return the model-facing response.""" + if not user_id: + return {"results": [], "result_count": 0, "error": SIGNED_OUT_MESSAGE} + origin = get_action_origin(manifest) + # Like MCP, require the origin bound by the authorized global-action lookup; a copied or + # rebuilt manifest without one can't prove it is an administrator-managed global action. + if origin is None or origin.scope_type != "global": + log_event( + "[AZURE_FILES_SEARCH] Refused an action that is not a bound global action.", + level=logging.WARNING, + extra={ + "action_id": str((manifest or {}).get("id") or ""), + "scope_type": origin.scope_type if origin is not None else "unbound", + }, + ) + return {"results": [], "result_count": 0, "error": MISCONFIGURED_MESSAGE} + try: + config = normalize_azure_files_search_config(manifest) + assert_not_internal_index(config, settings) + except AzureFilesSearchConfigError as error: + log_event( + "[AZURE_FILES_SEARCH] Action configuration rejected at run time.", + level=logging.WARNING, + extra={"action_id": str((manifest or {}).get("id") or ""), "reason": error.public_message}, + ) + return {"results": [], "result_count": 0, "error": MISCONFIGURED_MESSAGE} + + membership, identity_reason = (None, "") + if config.permission_mode != PERMISSION_MODE_NONE: + membership, identity_reason = build_user_membership(config, user_id) + dependencies = SearchDependencies( + run_search=_run_search_for(config), + read_file_sddl=read_file_sddl, + check_share_access=_check_share_access_for(user_id), + membership=membership, + evaluate_sddl=evaluate_sddl_read_access, + identity_error_reason=identity_reason, + ) + try: + outcome = run_azure_files_search(config, query, top_n, dependencies, vector_query_factory=VectorizableTextQuery) + except AzureFilesSearchUnavailable as error: + log_event( + "[AZURE_FILES_SEARCH] Search request failed.", + level=logging.WARNING, + extra={"action_id": config.action_id, "index_name": config.index_name, "error_code": error.code}, + ) + return {"results": [], "result_count": 0, "error": UNAVAILABLE_MESSAGE} + except Exception as error: + log_event( + "[AZURE_FILES_SEARCH] Search failed unexpectedly.", + level=logging.ERROR, + extra={"action_id": config.action_id, "exception_type": type(error).__name__}, + exceptionTraceback=True, + ) + return {"results": [], "result_count": 0, "error": UNAVAILABLE_MESSAGE} + + record_azure_files_search_review(config, user_id, outcome) + return build_model_response(config, outcome) + + +def _check(name: str, status: str, message: str) -> Dict[str, str]: + return {"name": name, "status": status, "message": message} + + +def check_azure_files_search_connection( + manifest: Dict[str, Any], + user_id: str, + settings: Dict[str, Any], +) -> Dict[str, Any]: + """Check an Azure Files Search configuration end to end for the administrator running the test.""" + checks: List[Dict[str, str]] = [] + try: + config = normalize_azure_files_search_config(manifest) + assert_not_internal_index(config, settings) + except AzureFilesSearchConfigError as error: + return {"success": False, "checks": [_check("configuration", "fail", error.public_message)]} + checks.append(_check("configuration", "pass", "The configuration is valid.")) + + sample_hits: List[Dict[str, Any]] = [] + try: + request = {"search_text": "*", "select": [config.path_field, config.name_field], "top": 25} + sample_hits = _run_search_for(config)(request) + checks.append(_check("search_access", "pass", f"SimpleChat queried the index and read {len(sample_hits)} sample documents.")) + except AzureFilesSearchUnavailable as error: + checks.append(_check("search_access", "fail", error.public_message)) + return {"success": False, "checks": checks} + + sample_paths = [parse_azure_file_path(hit.get(config.path_field)) for hit in sample_hits] + parsed_paths = [path for path in sample_paths if path is not None] + if not sample_hits: + checks.append(_check("file_paths", "warn", "The index returned no documents, so file paths couldn't be checked.")) + elif not parsed_paths: + checks.append(_check( + "file_paths", "fail", + f"The {config.path_field} field doesn't contain Azure Files URLs, so file permissions can't be checked.", + )) + else: + checks.append(_check("file_paths", "pass", f"{len(parsed_paths)} sample documents have Azure Files paths.")) + + if config.query_mode != "keyword": + try: + _run_search_for(config)( + {**_mode_probe_request(config), "top": 1} + ) + checks.append(_check("query_mode", "pass", f"{config.query_mode.capitalize()} search works on this index.")) + except AzureFilesSearchUnavailable as error: + checks.append(_check("query_mode", "fail", error.public_message)) + + if config.permission_mode == PERMISSION_MODE_NONE: + checks.append(_check( + "permissions", "warn", + "File permission checks are off. Everyone who can use this action can search every file in the index.", + )) + return {"success": all(check["status"] != "fail" for check in checks), "checks": checks} + + membership, identity_reason = build_user_membership(config, user_id) + if membership is None: + checks.append(_check( + "identity", "fail", + "SimpleChat couldn't read your group memberships from Microsoft Graph, so file permissions can't be checked. " + "Sign out and back in, and confirm the app registration has Group.Read.All consent.", + )) + else: + checks.append(_check("identity", "pass", "SimpleChat resolved your directory groups.")) + + for share in config.storage_shares: + label = f"{share.account.account_name}/{share.share_name}" + if config.share_access_check == SHARE_ACCESS_CHECK_RBAC: + state, _reason = _check_share_access_for(user_id)(share) + if state in (MEMBER, NOT_MEMBER): + checks.append(_check(f"share_access:{label}", "pass", f"SimpleChat read share-level permissions for {label}.")) + else: + checks.append(_check( + f"share_access:{label}", "fail", + f"SimpleChat couldn't read share-level permissions for {label}. Give its managed identity the Reader " + "role on the storage account, or turn off the share-level check.", + )) + share_path = next( + (path for path in parsed_paths if path.account_name == share.account.account_name and path.share_name == share.share_name), + None, + ) + if share_path is None: + checks.append(_check(f"file_permissions:{label}", "warn", f"No sample document came from {label}, so file permissions weren't read.")) + continue + try: + read_file_sddl(share_path) + checks.append(_check(f"file_permissions:{label}", "pass", f"SimpleChat read file permissions in {label}.")) + except AzureFileNotFound: + checks.append(_check(f"file_permissions:{label}", "warn", f"A sample file in {label} no longer exists. The index may be stale.")) + except Exception as error: + log_event( + "[AZURE_FILES_SEARCH] Connection test could not read file permissions.", + level=logging.WARNING, + extra={"exception_type": type(error).__name__, "storage_account": share.account.account_name}, + ) + checks.append(_check( + f"file_permissions:{label}", "fail", + f"SimpleChat couldn't read file permissions in {label}. Give its managed identity the Storage File Data " + "Privileged Reader role on the storage account or share, and allow it through the storage firewall.", + )) + + unlisted = sorted({ + f"{path.account_name}/{path.share_name}" for path in parsed_paths + if config.find_share(path.account_name, path.share_name) is None + }) + if unlisted: + checks.append(_check( + "storage_allowlist", "warn", + "The index contains files from shares that aren't listed, and those results will be withheld: " + ", ".join(unlisted[:5]), + )) + return {"success": all(check["status"] != "fail" for check in checks), "checks": checks} + + +def _mode_probe_request(config: AzureFilesSearchConfig) -> Dict[str, Any]: + request = build_search_request(config, "test", VectorizableTextQuery) + request["select"] = [config.path_field] + request.pop("highlight_fields", None) + request.pop("highlight_pre_tag", None) + request.pop("highlight_post_tag", None) + return request diff --git a/application/single_app/functions_file_sync.py b/application/single_app/functions_file_sync.py index d4a41d505..b398d74fe 100644 --- a/application/single_app/functions_file_sync.py +++ b/application/single_app/functions_file_sync.py @@ -14,7 +14,9 @@ from urllib.parse import parse_qsl, quote, unquote, urlparse from azure.core import MatchConditions +from azure.core.exceptions import ClientAuthenticationError as AzureClientAuthenticationError from azure.core.exceptions import ResourceNotFoundError as AzureResourceNotFoundError +from azure.core.exceptions import ServiceRequestError as AzureServiceRequestError from azure.identity import ClientSecretCredential, DefaultAzureCredential from azure.cosmos.exceptions import ( CosmosAccessConditionFailedError, @@ -45,6 +47,7 @@ from functions_azure_endpoint_validation import ( AZURE_STORAGE_ENDPOINT_SUFFIXES, azure_storage_endpoint_suffix_for_hostname, + validate_azure_file_endpoint, ) from functions_debug import debug_print from functions_documents import ( @@ -57,7 +60,7 @@ update_document, validate_tags, ) -from functions_group import assert_group_role +from functions_group import assert_group_role, find_group_by_id, get_group_document_reviewer_ids from functions_keyvault import ( keyvault_file_sync_cleanup_helper, keyvault_file_sync_delete_helper, @@ -143,6 +146,68 @@ FILE_SYNC_PERSONAL_APP_ROLE = "PersonalFileSyncUser" FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE = "File Sync run failed. Contact an administrator if the problem continues." FILE_SYNC_PUBLIC_ITEM_ERROR_MESSAGE = "File Sync could not process this item. Contact an administrator if the problem continues." +# Azure Files OAuth over REST accepts token credentials only with backup intent, and the +# identity then needs Storage File Data Privileged Reader, which reads every file in the +# share regardless of its NTFS permissions. +AZURE_FILES_TOKEN_INTENT = "backup" +AZURE_FILES_ERROR_PERMISSION_DENIED = "azure_files_permission_denied" +AZURE_FILES_ERROR_AUTHENTICATION_FAILED = "azure_files_authentication_failed" +AZURE_FILES_ERROR_NOT_FOUND = "azure_files_not_found" +AZURE_FILES_ERROR_NETWORK_BLOCKED = "azure_files_network_blocked" +AZURE_FILES_ENDPOINT_PUBLIC_ERROR = ( + "Azure Files sources require an Azure Files service URL such as https://account.file.core.windows.net." +) +# Reviewed, static messages that are safe to return for a classified run or connection failure. +FILE_SYNC_RUN_ERROR_CATEGORY_MESSAGES = { + AZURE_FILES_ERROR_PERMISSION_DENIED: ( + "Azure Files denied access. Give the managed identity or service principal this source uses " + "the Storage File Data Privileged Reader role on the storage account or file share (a SAS needs " + "Read and List), and confirm the storage account's network rules allow this app." + ), + AZURE_FILES_ERROR_AUTHENTICATION_FAILED: ( + "Azure Files could not authenticate this source's credential. Check its managed identity, " + "service principal, or connection string." + ), + AZURE_FILES_ERROR_NOT_FOUND: ( + "Azure Files could not find the file share or directory. Check the share name and directory path." + ), + AZURE_FILES_ERROR_NETWORK_BLOCKED: ( + "Azure Files blocked the request because of a network restriction. Allow this app's outbound " + "addresses or private endpoint in the storage account's networking settings." + ), +} +FILE_SYNC_RUN_FAILED_NOTIFICATION_TYPE = "file_sync_run_failed" +# Only an Azure role assignment or credential fix resolves these, so app admins are told as well. +FILE_SYNC_ADMIN_NOTIFICATION_ERROR_CATEGORIES = { + AZURE_FILES_ERROR_PERMISSION_DENIED, + AZURE_FILES_ERROR_AUTHENTICATION_FAILED, +} +FILE_SYNC_RUN_FAILED_NOTIFICATION_MAX_RECIPIENTS = 25 +_AZURE_FILES_PERMISSION_ERROR_CODES = { + "authorizationpermissionmismatch", + "authorizationfailure", + "insufficientaccountpermissions", + "authorizationresourcetypemismatch", + "authorizationservicemismatch", +} +_AZURE_FILES_AUTHENTICATION_ERROR_CODES = { + "authenticationfailed", + "invalidauthenticationinfo", + "noauthenticationinformation", +} +_AZURE_FILES_NOT_FOUND_ERROR_CODES = { + "sharenotfound", + "resourcenotfound", + "parentnotfound", + "sharedisabled", + "sharebeingdeleted", +} +_AZURE_FILES_NETWORK_ERROR_CODES = { + "authorizationsourceipmismatch", + "authorizationipmismatch", + "ipauthorizationfailure", + "ipsourcenotallowed", +} class FileSyncPublicValidationError(ValueError): @@ -744,7 +809,12 @@ def sanitize_file_sync_source(source: Dict[str, Any]) -> Dict[str, Any]: def sanitize_file_sync_run(run: Dict[str, Any]) -> Dict[str, Any]: sanitized_run = dict(run or {}) if sanitized_run.get("error_message"): - sanitized_run["error_message"] = FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE + sanitized_run["error_message"] = FILE_SYNC_RUN_ERROR_CATEGORY_MESSAGES.get( + sanitized_run.get("error_category") or "", + FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE, + ) + if sanitized_run.get("error_category") not in FILE_SYNC_RUN_ERROR_CATEGORY_MESSAGES: + sanitized_run.pop("error_category", None) return sanitized_run @@ -894,9 +964,15 @@ def _normalize_azure_file_url(value: Any) -> Tuple[str, List[str]]: parsed_url = urlparse(raw_url) if parsed_url.scheme != "https" or not parsed_url.netloc: raise ValueError("Azure Files sources require an HTTPS file service or share URL") + # Managed identity and service principal sources send a storage token to this host, + # so only Azure Files service endpoints are accepted. + try: + account_url = validate_azure_file_endpoint(f"{parsed_url.scheme}://{parsed_url.netloc}") + except ValueError as error: + raise FileSyncPublicValidationError(AZURE_FILES_ENDPOINT_PUBLIC_ERROR) from error path_parts = [unquote(path_part) for path_part in parsed_url.path.split("/") if path_part] - return f"{parsed_url.scheme}://{parsed_url.netloc}".rstrip("/"), path_parts + return account_url, path_parts def _normalize_azure_share_name(value: Any) -> str: @@ -2286,10 +2362,62 @@ def _test_azure_files_connection(source: Dict[str, Any]) -> Dict[str, Any]: if "azure-storage-file-share" in str(error): raise raise ValueError("Azure Files connection test failed. Verify the file endpoint, share, and identity permissions.") from error + except FileSyncPublicValidationError: + raise except Exception as error: + error_category = classify_azure_files_error(error) + log_event( + "[FILE_SYNC] Azure Files connection test failed.", + level=logging.WARNING, + extra={**_azure_files_error_diagnostics(error, source), "error_category": error_category}, + exceptionTraceback=True, + ) + if error_category: + raise FileSyncPublicValidationError(FILE_SYNC_RUN_ERROR_CATEGORY_MESSAGES[error_category]) from error raise ValueError("Azure Files connection test failed. Verify the file endpoint, share, and identity permissions.") from error +def classify_azure_files_error(error: Exception) -> str: + """Map an Azure Files or credential failure to a reviewed error category, or ``""``.""" + error_code = _normalize_azure_storage_error_code(getattr(error, "error_code", "")) + response = getattr(error, "response", None) + status_code = getattr(error, "status_code", None) or getattr(response, "status_code", None) + if error_code in _AZURE_FILES_NETWORK_ERROR_CODES: + return AZURE_FILES_ERROR_NETWORK_BLOCKED + if error_code in _AZURE_FILES_PERMISSION_ERROR_CODES: + return AZURE_FILES_ERROR_PERMISSION_DENIED + if error_code in _AZURE_FILES_AUTHENTICATION_ERROR_CODES: + return AZURE_FILES_ERROR_AUTHENTICATION_FAILED + if error_code in _AZURE_FILES_NOT_FOUND_ERROR_CODES or isinstance(error, AzureResourceNotFoundError): + return AZURE_FILES_ERROR_NOT_FOUND + if status_code == 403: + return AZURE_FILES_ERROR_PERMISSION_DENIED + if status_code == 401: + return AZURE_FILES_ERROR_AUTHENTICATION_FAILED + if status_code == 404: + return AZURE_FILES_ERROR_NOT_FOUND + # Token acquisition failures (managed identity or service principal) carry no storage error code. + if isinstance(error, AzureClientAuthenticationError): + return AZURE_FILES_ERROR_AUTHENTICATION_FAILED + if isinstance(error, AzureServiceRequestError): + return AZURE_FILES_ERROR_NETWORK_BLOCKED + return "" + + +def _azure_files_error_diagnostics(error: Exception, source: Dict[str, Any]) -> Dict[str, Any]: + response = getattr(error, "response", None) + headers = getattr(response, "headers", {}) or {} + auth = _get_identity_auth_for_source(source) or source.get("auth") or {} + return { + "exception_type": type(error).__name__, + "error_code": _normalize_azure_storage_error_code(getattr(error, "error_code", "")), + "status_code": getattr(error, "status_code", None) or getattr(response, "status_code", None), + "request_id": _normalize_text(headers.get("x-ms-request-id", ""), 100), + "source_id": source.get("id"), + "auth_kind": _normalize_text(auth.get("auth_type"), 50).lower(), + } + + def _test_azure_blob_connection(source: Dict[str, Any]) -> Dict[str, Any]: try: container_client = _get_azure_blob_container_client(source) @@ -2816,21 +2944,27 @@ def _process_file_sync_source( return run except Exception as error: detailed_error_message = str(error) - run = _update_run( - run, - { - "status": "failed", - "counts": counts, - "completed_at": _now_iso(), - "error_message": FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE, - }, + error_category = ( + classify_azure_files_error(error) + if source.get("source_type") == FILE_SYNC_SOURCE_TYPE_AZURE_FILES + else "" ) + public_error_message = FILE_SYNC_RUN_ERROR_CATEGORY_MESSAGES.get(error_category, FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE) + run_changes = { + "status": "failed", + "counts": counts, + "completed_at": _now_iso(), + "error_message": public_error_message, + } + if error_category: + run_changes["error_category"] = error_category + run = _update_run(run, run_changes) _update_source_after_run(source, run) _log_file_sync_activity( source, triggered_by, "run_failed", - {"run_id": run["id"], "error": FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE}, + {"run_id": run["id"], "error": public_error_message, "error_category": error_category}, ) log_event( "[FILE_SYNC] Run failed.", @@ -2839,9 +2973,11 @@ def _process_file_sync_source( "source_id": source.get("id"), "run_id": run.get("id"), "error": detailed_error_message, + "error_category": error_category, }, exceptionTraceback=True, ) + _notify_file_sync_run_failed(source, run, error_category) return run @@ -3255,6 +3391,8 @@ def _get_azure_files_service_client(source: Dict[str, Any]): account_url = connection.get("account_url") or "" if not account_url: raise ValueError("Azure Files source is missing an account URL") + # Re-checked here so sources saved before the host check can't send a token elsewhere. + account_url = validate_azure_file_endpoint(account_url) if auth_type == "client_secret": client_id = auth.get("identity") or "" client_secret = _resolved_auth_secret(auth) @@ -3270,7 +3408,8 @@ def _get_azure_files_service_client(source: Dict[str, Any]): credential = DefaultAzureCredential(managed_identity_client_id=auth.get("managed_identity_client_id") or None) else: raise ValueError("Azure Files sources require managed identity, service principal, or connection string authentication") - return ShareServiceClient(account_url=account_url, credential=credential) + # Token credentials must declare backup intent; the SDK refuses to build the client otherwise. + return ShareServiceClient(account_url=account_url, credential=credential, token_intent=AZURE_FILES_TOKEN_INTENT) def _get_azure_files_share_client(source: Dict[str, Any]): @@ -4411,4 +4550,87 @@ def _log_file_sync_activity(source: Dict[str, Any], user_id: Optional[str], acti additional_context=additional_context or {}, ) except Exception as error: - log_event(f"[FILE_SYNC] Failed to log activity: {error}", level=logging.WARNING) \ No newline at end of file + log_event(f"[FILE_SYNC] Failed to log activity: {error}", level=logging.WARNING) + + +def _file_sync_manager_recipient_ids(source: Dict[str, Any]) -> List[str]: + """Return the users who currently manage a source's workspace, capped for fan-out.""" + scope_type = source.get("scope_type") + scope_id = _source_scope_id(source) + candidates: List[Any] = [] + if scope_type == FILE_SYNC_SCOPE_PERSONAL: + candidates = [scope_id] + elif scope_type == FILE_SYNC_SCOPE_GROUP: + candidates = get_group_document_reviewer_ids(find_group_by_id(scope_id)) + elif scope_type == FILE_SYNC_SCOPE_PUBLIC: + workspace_doc = find_public_workspace_by_id(scope_id) or {} + members = [*(workspace_doc.get("admins") or []), *(workspace_doc.get("documentManagers") or [])] + candidates = [(workspace_doc.get("owner") or {}).get("userId")] + candidates.extend(member.get("userId") if isinstance(member, dict) else member for member in members) + candidates = [ + user_id for user_id in candidates + if isinstance(user_id, str) + and get_user_role_in_public_workspace(workspace_doc, user_id) in FILE_SYNC_MANAGER_ROLES + ] + recipient_ids = sorted({user_id for user_id in candidates if isinstance(user_id, str) and user_id}) + return recipient_ids[:FILE_SYNC_RUN_FAILED_NOTIFICATION_MAX_RECIPIENTS] + + +def _file_sync_workspace_link(source: Dict[str, Any]) -> str: + scope_type = source.get("scope_type") + if scope_type == FILE_SYNC_SCOPE_GROUP: + return "/group_workspaces" + if scope_type == FILE_SYNC_SCOPE_PUBLIC: + return f"/public_workspaces/{quote(str(_source_scope_id(source)), safe='')}" + return "/workspace" + + +def _notify_file_sync_run_failed(source: Dict[str, Any], run: Dict[str, Any], error_category: str) -> None: + """Tell a failed source's managers once a day, and app admins when Azure access must change.""" + try: + # Imported lazily, like the activity-logging hook: functions_notifications pulls in group, + # public workspace and workflow alert modules, and File Sync also runs in the scheduler. + from functions_notifications import create_notification + + source_id = str(source.get("id") or "") + if not source_id: + return + source_name = _normalize_text(source.get("name"), 200) or "File Sync source" + reason = FILE_SYNC_RUN_ERROR_CATEGORY_MESSAGES.get(error_category, FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE) + day = _now().strftime("%Y-%m-%d") + metadata = { + "source_id": source_id, + "run_id": run.get("id"), + "scope_type": source.get("scope_type"), + "source_type": source.get("source_type"), + "error_category": error_category or "unclassified", + } + link_context = {"workspace_type": source.get("scope_type"), "source_id": source_id} + for recipient_id in _file_sync_manager_recipient_ids(source): + create_notification( + user_id=recipient_id, + notification_type=FILE_SYNC_RUN_FAILED_NOTIFICATION_TYPE, + title=f"File Sync failed: {source_name}", + message=reason, + link_url=_file_sync_workspace_link(source), + link_context=link_context, + metadata=metadata, + idempotency_key=f"file-sync-run-failed:{source_id}:{day}", + ) + if error_category in FILE_SYNC_ADMIN_NOTIFICATION_ERROR_CATEGORIES: + create_notification( + notification_type=FILE_SYNC_RUN_FAILED_NOTIFICATION_TYPE, + title=f"File Sync needs Azure access: {source_name}", + message=reason, + link_url="/admin/settings", + link_context=link_context, + metadata=metadata, + assignment={"roles": ["Admin"]}, + idempotency_key=f"file-sync-run-failed-admin:{source_id}:{day}", + ) + except Exception as error: + log_event( + "[FILE_SYNC] Failed to create run failure notification.", + level=logging.WARNING, + extra={"source_id": source.get("id"), "exception_type": type(error).__name__}, + ) \ No newline at end of file diff --git a/application/single_app/functions_global_actions.py b/application/single_app/functions_global_actions.py index 33c2cc831..f5285a52f 100644 --- a/application/single_app/functions_global_actions.py +++ b/application/single_app/functions_global_actions.py @@ -15,6 +15,7 @@ from functions_action_manifest import McpConfigurationError, bind_action_origin from functions_appinsights import log_event from functions_authentication import get_current_user_id +from functions_azure_files_search import AZURE_FILES_INDEX_ACTION_TYPE, validate_azure_files_index_action from functions_keyvault import keyvault_plugin_save_helper, keyvault_plugin_get_helper, keyvault_plugin_delete_helper, SecretReturnType from functions_workspace_identities import ( WORKSPACE_IDENTITY_SCOPE_GLOBAL, @@ -137,6 +138,8 @@ def save_global_action(action_data, user_id=None): submitted_action = action_data action_data = normalize_m365_action_payload(action_data) action_data = prepare_scoped_action(action_data, "global", "global") + if action_data.get("type") == AZURE_FILES_INDEX_ACTION_TYPE: + validate_azure_files_index_action(action_data, get_settings()) if user_id is None: user_id = get_current_user_id() actor_user_id = user_id diff --git a/application/single_app/functions_legacy_action_management.py b/application/single_app/functions_legacy_action_management.py index de034fce9..2e31670aa 100644 --- a/application/single_app/functions_legacy_action_management.py +++ b/application/single_app/functions_legacy_action_management.py @@ -8,11 +8,13 @@ import json from functions_action_manifest import ( + GlobalOnlyActionTypeError, McpConfigurationError, McpStdioRemovedError, bind_action_origin, get_action_execution_status, get_action_origin, + is_global_only_action_type, is_retired_mcp_stdio, resolve_action_type, ) @@ -199,6 +201,8 @@ def prepare_scoped_action(action, scope_type, scope_id): payload = bind_action_origin(payload, scope_type, scope_id) if is_retired_mcp_stdio(payload): raise McpStdioRemovedError() + if scope_type != "global" and is_global_only_action_type(payload.get("type")): + raise GlobalOnlyActionTypeError() if payload["type"] == "mcp": # Save-time normalization is deferred so management-only inspection never # loads preset catalogs or executable action dependencies during bootstrap. diff --git a/application/single_app/functions_notifications.py b/application/single_app/functions_notifications.py index cd13df387..94de5b4f8 100644 --- a/application/single_app/functions_notifications.py +++ b/application/single_app/functions_notifications.py @@ -261,6 +261,14 @@ KEY_VAULT_SECRET_REMINDER_NOTIFICATION_TYPE: { 'icon': 'bi-safe', 'color': 'warning' + }, + 'file_sync_run_failed': { + 'icon': 'bi-cloud-slash', + 'color': 'danger' + }, + 'azure_files_search_access_unverified': { + 'icon': 'bi-shield-exclamation', + 'color': 'warning' } } diff --git a/application/single_app/route_backend_control_center.py b/application/single_app/route_backend_control_center.py index c0bc7194c..d69b38c72 100644 --- a/application/single_app/route_backend_control_center.py +++ b/application/single_app/route_backend_control_center.py @@ -1053,6 +1053,25 @@ def format_activity_log_details_for_csv(log_record): detail_parts.append(f"Error: {additional_context.get('error')}") return '; '.join(detail_parts) + if activity_type == 'azure_files_search_access': + action_context = log_record.get('action_context', {}) or {} + additional_context = log_record.get('additional_context', {}) or {} + counts = additional_context.get('counts', {}) or {} + agent = log_record.get('agent') or {} + reasons = '; '.join( + f"{reason}: {count}" for reason, count in (additional_context.get('reasons') or {}).items() + ) or 'N/A' + return '; '.join([ + f"Action: {action_context.get('display_name') or action_context.get('action_name') or 'Azure Files Search'}", + f"Index: {action_context.get('index_name') or 'N/A'}", + f"Agent: {agent.get('display_name') or agent.get('name') or 'N/A'}", + f"Conversation: {log_record.get('conversation_id') or 'N/A'}", + f"Denied: {counts.get('denied_files', 0)}", + f"Unverified: {counts.get('unverified_files', 0)}", + f"Files checked: {counts.get('files_evaluated', 0)}", + f"Reasons: {reasons}", + ]) + if activity_type == 'data_management': workspace_context = log_record.get('workspace_context', {}) additional_context = log_record.get('additional_context', {}) diff --git a/application/single_app/route_backend_group_actions_scoped.py b/application/single_app/route_backend_group_actions_scoped.py index 3f4485891..15b93b49a 100644 --- a/application/single_app/route_backend_group_actions_scoped.py +++ b/application/single_app/route_backend_group_actions_scoped.py @@ -21,6 +21,7 @@ from flask import jsonify, request +from functions_action_manifest import is_global_only_action_type from functions_authentication import login_required, user_required, get_current_user_id from functions_settings import enabled_required from functions_group_action_access import ( @@ -139,7 +140,8 @@ def api_scoped_group_actions_types(group_id): # the editor catalogue the V2 editor renders (auth types and field schemas), # never the raw discovery Response, and keep it out of shared caches. discovered = get_plugin_types( - allowed_type_filter=lambda action_type: is_action_type_access_allowed( + allowed_type_filter=lambda action_type: not is_global_only_action_type(action_type) + and is_action_type_access_allowed( 'governance_group_actions', user_id, action_type, 'group', ), ) diff --git a/application/single_app/route_backend_plugins.py b/application/single_app/route_backend_plugins.py index 6933335c0..7e3ed000a 100644 --- a/application/single_app/route_backend_plugins.py +++ b/application/single_app/route_backend_plugins.py @@ -29,12 +29,14 @@ from functions_authentication import * from functions_appinsights import log_event from functions_action_manifest import ( + GlobalOnlyActionTypeError, McpActionOrigin, McpConfigurationError, McpStdioRemovedError, ScopedActionManifest, bind_action_origin, get_action_origin, + is_global_only_action_type, is_retired_mcp_stdio, resolve_action_type, ) @@ -89,6 +91,12 @@ from functions_debug import debug_print from functions_azure_endpoint_validation import validate_azure_cosmos_endpoint +from functions_azure_files_search import ( + AZURE_FILES_INDEX_ACTION_TYPE, + AzureFilesSearchConfigError, + validate_azure_files_index_action, +) +from functions_azure_files_search_runtime import check_azure_files_search_connection from json_schema_validation import ( PLUGIN_STORAGE_MANAGED_FIELDS, apply_plugin_validation_defaults, @@ -1443,6 +1451,8 @@ def _prepare_personal_action_payload(user_id, plugin, *, editor=False, existing= plugin_to_save.setdefault('endpoint', '') _apply_plugin_runtime_defaults(plugin_to_save) plugin_type = plugin_to_save['type'] + if is_global_only_action_type(plugin_type): + return None, (jsonify({'error': GlobalOnlyActionTypeError.public_message}), 403) try: _validate_action_identity_for_scope( plugin_to_save, @@ -1520,6 +1530,8 @@ def _prepare_group_action_payload(user_id, group_id, plugin, settings, existing) plugin_to_save.setdefault('endpoint', '') _apply_plugin_runtime_defaults(plugin_to_save) plugin_type = plugin_to_save['type'] + if is_global_only_action_type(plugin_type): + return None, (jsonify({'error': GlobalOnlyActionTypeError.public_message}), 403) try: _validate_action_identity_for_scope( plugin_to_save, @@ -1623,6 +1635,12 @@ def _prepare_global_action_payload(user_id, plugin, settings, existing): is_valid, validation_errors = PluginHealthChecker.validate_plugin_manifest(plugin_to_save, plugin_type) if not is_valid: return None, (jsonify({'error': f'Plugin validation failed: {"; ".join(validation_errors)}'}), 400) + if plugin_type == AZURE_FILES_INDEX_ACTION_TYPE: + # Same save-time checks as the classic admin route, including refusing SimpleChat's own indexes. + try: + validate_azure_files_index_action(plugin_to_save, settings) + except AzureFilesSearchConfigError as exc: + return None, (jsonify({'error': f'Plugin validation failed: {exc.public_message}'}), 400) try: validate_mcp_tool_pinning_for_save(plugin_to_save, existing) except ValueError: @@ -2321,7 +2339,8 @@ def get_user_plugin_types(): try: ensure_editor_access('actions', user_id, get_settings(), operation='read') discovered = get_plugin_types( - allowed_type_filter=lambda action_type: is_action_type_access_allowed( + allowed_type_filter=lambda action_type: not is_global_only_action_type(action_type) + and is_action_type_access_allowed( 'governance_user_actions', user_id, action_type, 'personal', ), ) @@ -2331,7 +2350,8 @@ def get_user_plugin_types(): except Exception as exc: return editor_error_response(exc) return get_plugin_types( - allowed_type_filter=lambda action_type: is_action_type_access_allowed( + allowed_type_filter=lambda action_type: not is_global_only_action_type(action_type) + and is_action_type_access_allowed( 'governance_user_actions', user_id, action_type, @@ -2360,7 +2380,8 @@ def get_group_plugin_types(): return jsonify({'error': 'You are not authorized to list group action types.'}), 403 return get_plugin_types( - allowed_type_filter=lambda action_type: is_action_type_access_allowed( + allowed_type_filter=lambda action_type: not is_global_only_action_type(action_type) + and is_action_type_access_allowed( 'governance_group_actions', user_id, action_type, @@ -4281,3 +4302,38 @@ def test_snowflake_action_connection(): def test_tableau_action_connection(): """Test a Tableau action by signing in to the configured server and site.""" return _run_action_connection_test(TABLEAU_PLUGIN_TYPE, 'Tableau', test_tableau_connection) + + +def _require_global_azure_files_index_test(manifest, scope_type, scope_id): + """Azure Files Search runs with the application identity, so only global actions are tested.""" + if scope_type != 'global': + raise PermissionError('Azure Files Search actions are global actions.') + + +def _check_azure_files_index_action(manifest): + result = check_azure_files_search_connection(dict(manifest), get_current_user_id(), get_settings()) + checks = result.get('checks') or [] + # Warn-level checks stay in the ordered checks list; both editors render that list with status badges. + response = {'details': {'checks': checks}} + failures = [check for check in checks if check.get('status') == 'fail'] + if failures: + return {**response, 'success': False, 'error': failures[0]['message'], 'status': 400} + return { + **response, + 'success': True, + 'message': 'Azure Files Search can query the index and check file permissions.', + } + + +@bpap.route('/api/plugins/test-azure-files-index-connection', methods=['POST']) +@swagger_route(security=get_auth_security()) +@login_required +@admin_required +def test_azure_files_index_action_connection(): + """Test an Azure Files Search action: index access, file paths, file permissions, share access, and identity.""" + return _run_action_connection_test( + AZURE_FILES_INDEX_ACTION_TYPE, + 'Azure Files Search', + _check_azure_files_index_action, + before_test=_require_global_azure_files_index_test, + ) diff --git a/application/single_app/semantic_kernel_plugins/azure_files_index_plugin.py b/application/single_app/semantic_kernel_plugins/azure_files_index_plugin.py new file mode 100644 index 000000000..84f47f118 --- /dev/null +++ b/application/single_app/semantic_kernel_plugins/azure_files_index_plugin.py @@ -0,0 +1,91 @@ +# azure_files_index_plugin.py +"""Semantic Kernel action that searches an Azure Files indexer index with live file permissions. + +The action queries a customer's existing Azure AI Search index that was built by the Azure Files +indexer, then returns only results from files the signed-in user can open: each candidate file's +NTFS DACL and the share's permissions are checked against the user's directory identity before +anything reaches the model. Withheld files are never mentioned to the user; administrators see +them in the activity logs. +""" + +from typing import Annotated, Any, Dict, List, Optional + +from semantic_kernel.functions import kernel_function + +from functions_authentication import get_current_user_id +from functions_azure_files_search import ( + AZURE_FILES_INDEX_ACTION_TYPE, + AZURE_FILES_SEARCH_DISPLAY_NAME, + MAX_TOP_N, +) +from functions_azure_files_search_runtime import execute_azure_files_search +from functions_settings import get_settings +from semantic_kernel_plugins.base_plugin import BasePlugin +from semantic_kernel_plugins.plugin_invocation_logger import plugin_function_logger + + +class AzureFilesIndexPlugin(BasePlugin): + """Search an Azure Files indexer index and keep only files the user can open.""" + + def __init__(self, manifest: Optional[Dict[str, Any]] = None): + super().__init__(manifest) + self.manifest = manifest or {} + metadata = self.manifest.get("metadata") + self._metadata = metadata if isinstance(metadata, dict) else {} + + @property + def display_name(self) -> str: + return AZURE_FILES_SEARCH_DISPLAY_NAME + + @property + def metadata(self) -> Dict[str, Any]: + user_description = self._metadata.get( + "description", + "Searches file share content indexed by the Azure AI Search Azure Files indexer.", + ) + api_description = ( + "Search an Azure AI Search index that was built from Azure file shares. Results include only " + "files the signed-in user can open on the share, each with its file name and network path." + ) + return { + "name": self._metadata.get("name", "azure_files_index_plugin"), + "type": AZURE_FILES_INDEX_ACTION_TYPE, + "description": f"{user_description}\n\n{api_description}", + "methods": [ + { + "name": "search_files", + "description": ( + "Search the file share index for passages relevant to a question. Returns only files the " + "signed-in user can open, with file names and network paths to cite." + ), + "parameters": [ + {"name": "query", "type": "str", "description": "Natural-language search query.", "required": True}, + {"name": "top_n", "type": "int", "description": f"Maximum results to return (1-{MAX_TOP_N}).", "required": False}, + ], + "returns": {"type": "dict", "description": "Matching passages with file names and network paths."}, + }, + ], + } + + def get_functions(self) -> List[str]: + return ["search_files"] + + # bac-check: ignore - the user is the signed-in session user; the model supplies only query text and a bounded count. + @plugin_function_logger("AzureFilesIndexPlugin") + @kernel_function( + name="search_files", + description=( + "Search the configured file share index for passages relevant to the question. Returns only files the " + "signed-in user can open. Cite each result by its file_name and unc_path." + ), + ) + def search_files( + self, + query: Annotated[str, "Natural-language query describing what to find in the file shares."], + top_n: Annotated[int, f"Maximum number of results to return, 1 to {MAX_TOP_N}. Use 0 for the default."] = 0, + ) -> Annotated[dict, "Matching passages with file names and network paths."]: + try: + user_id = get_current_user_id() + except RuntimeError: + user_id = None + return execute_azure_files_search(self.manifest, query, top_n or None, user_id, get_settings()) diff --git a/application/single_app/semantic_kernel_plugins/plugin_health_checker.py b/application/single_app/semantic_kernel_plugins/plugin_health_checker.py index 39e77a053..c56882119 100644 --- a/application/single_app/semantic_kernel_plugins/plugin_health_checker.py +++ b/application/single_app/semantic_kernel_plugins/plugin_health_checker.py @@ -24,6 +24,11 @@ validate_azure_monitor_query_endpoint, validate_azure_queue_endpoint, ) +from functions_azure_files_search import ( + AZURE_FILES_INDEX_ACTION_TYPE, + AzureFilesSearchConfigError, + normalize_azure_files_search_config, +) from functions_azure_maps import AZURE_MAPS_DEFAULT_ENDPOINT, AZURE_MAPS_PLUGIN_TYPE from functions_blob_storage_operations import ( BLOB_STORAGE_PLUGIN_TYPE, @@ -455,6 +460,12 @@ def validate_plugin_manifest(manifest: Dict[str, Any], plugin_type: str) -> Tupl if auth_type == 'key' and not auth.get('key'): errors.append("Cosmos plugin requires auth.key when auth.type='key'") + elif plugin_type == AZURE_FILES_INDEX_ACTION_TYPE: + try: + normalize_azure_files_search_config(manifest) + except AzureFilesSearchConfigError as exc: + errors.append(exc.public_message) + elif plugin_type == ROCKSDB_PLUGIN_TYPE: errors.extend(PluginHealthChecker._validate_rocksdb_manifest(manifest)) diff --git a/application/single_app/static/js/control-center.js b/application/single_app/static/js/control-center.js index 2c11536c3..b23a3d939 100644 --- a/application/single_app/static/js/control-center.js +++ b/application/single_app/static/js/control-center.js @@ -2801,6 +2801,7 @@ class ControlCenter { 'document_deletion': 'Document Deleted', 'document_metadata_update': 'Document Metadata Updated', 'file_sync': 'File Sync', + 'azure_files_search_access': 'Azure Files Search Access', 'data_management': 'Data Management', 'token_usage': 'Token Usage', 'group_status_change': 'Group Status Change', @@ -2936,6 +2937,20 @@ class ControlCenter { } return `Action: ${this.escapeHtml(fileSyncAction)}
Source: ${this.escapeHtml(fileSyncSource)}
Scope: ${this.escapeHtml(fileSyncScope)}${fileSyncDetails.length ? ' · ' + fileSyncDetails.join(' · ') : ''}`; + case 'azure_files_search_access': { + const reviewAction = log.action_context?.display_name || log.action_context?.action_name || 'Azure Files Search'; + const reviewAgent = log.agent?.display_name || log.agent?.name || ''; + const reviewCounts = log.additional_context?.counts || {}; + const reviewReasons = Object.entries(log.additional_context?.reasons || {}) + .map(([reason, count]) => `${this.formatActivityValue(reason)}: ${this.formatActivityLogNumber(count)}`); + const reviewSummary = `Withheld ${this.formatActivityLogNumber(reviewCounts.denied_files || 0)} denied and ` + + `${this.formatActivityLogNumber(reviewCounts.unverified_files || 0)} unverified of ` + + `${this.formatActivityLogNumber(reviewCounts.files_evaluated || 0)} files`; + return `Action: ${this.escapeHtml(reviewAction)}` + + `${reviewAgent ? ` · Agent: ${this.escapeHtml(reviewAgent)}` : ''}
${this.escapeHtml(reviewSummary)}` + + `${reviewReasons.length ? `
${this.escapeHtml(reviewReasons.join(' · '))}` : ''}`; + } + case 'data_management': const dataManagementContext = log.additional_context || {}; const dataManagementAction = this.formatActivityValue(log.action || 'data_management_event'); @@ -3223,6 +3238,23 @@ class ControlCenter { case 'document_creation': return `File: ${this.escapeHtml(log.document?.file_name || 'Unknown')}, Type: ${this.escapeHtml(log.document?.file_type || '')}`; + case 'azure_files_search_access': { + const counts = log.additional_context?.counts || {}; + const reasons = Object.entries(log.additional_context?.reasons || {}) + .map(([reason, count]) => `${reason}: ${count}`) + .join('; '); + return [ + `Action: ${this.escapeHtml(log.action_context?.display_name || log.action_context?.action_name || 'Azure Files Search')}`, + `Index: ${this.escapeHtml(log.action_context?.index_name || 'N/A')}`, + `Agent: ${this.escapeHtml(log.agent?.display_name || log.agent?.name || 'N/A')}`, + `Conversation: ${this.escapeHtml(log.conversation_id || 'N/A')}`, + `Denied: ${this.escapeHtml(String(counts.denied_files ?? 0))}`, + `Unverified: ${this.escapeHtml(String(counts.unverified_files ?? 0))}`, + `Files checked: ${this.escapeHtml(String(counts.files_evaluated ?? 0))}`, + `Reasons: ${this.escapeHtml(reasons || 'N/A')}` + ].join(', '); + } + case 'file_sync': { const workspaceContext = log.workspace_context || {}; const additionalContext = log.additional_context || {}; @@ -3555,6 +3587,50 @@ class ControlCenter { } ); break; + case 'azure_files_search_access': + summaryFields.push( + { + label: 'Action', + value: log.action_context?.display_name || log.action_context?.action_name || 'Azure Files Search', + columnClass: 'col-md-6' + }, + { + label: 'Index', + value: log.action_context?.index_name, + columnClass: 'col-md-6' + }, + { + label: 'Agent', + value: log.agent?.display_name || log.agent?.name, + columnClass: 'col-md-6' + }, + { + label: 'Conversation', + value: log.conversation_id, + columnClass: 'col-md-6' + }, + { + label: 'Denied Files', + value: log.additional_context?.counts?.denied_files, + formatter: (value) => this.formatActivityLogNumber(value), + badgeClass: 'bg-secondary', + columnClass: 'col-md-4' + }, + { + label: 'Unverified Files', + value: log.additional_context?.counts?.unverified_files, + formatter: (value) => this.formatActivityLogNumber(value), + badgeClass: 'bg-warning text-dark', + columnClass: 'col-md-4' + }, + { + label: 'Files Checked', + value: log.additional_context?.counts?.files_evaluated, + formatter: (value) => this.formatActivityLogNumber(value), + columnClass: 'col-md-4' + } + ); + break; case 'conversation_creation': case 'conversation_deletion': case 'conversation_archival': diff --git a/application/single_app/static/js/plugin_modal_stepper.js b/application/single_app/static/js/plugin_modal_stepper.js index c6b3dd196..39fbe1b71 100644 --- a/application/single_app/static/js/plugin_modal_stepper.js +++ b/application/single_app/static/js/plugin_modal_stepper.js @@ -87,9 +87,49 @@ const ROCKSDB_DEFAULT_TIMEOUT = 30; const LOG_ANALYTICS_PLUGIN_TYPE = 'log_analytics'; const LOG_ANALYTICS_DEFAULT_ENDPOINT = 'https://api.loganalytics.io'; const LOG_ANALYTICS_DEFAULT_CLOUD = 'public'; +const AZURE_FILES_INDEX_PLUGIN_TYPE = 'azure_files_index'; +const AZURE_FILES_INDEX_FIELD_DEFAULTS = { + document_per_file: { + content_field: 'content', + title_field: 'metadata_storage_name', + path_field: 'metadata_storage_path', + name_field: 'metadata_storage_name', + last_modified_field: 'metadata_storage_last_modified', + vector_field: '', + select_content: false + }, + chunked: { + content_field: 'chunk', + title_field: 'title', + path_field: 'metadata_storage_path', + name_field: 'metadata_storage_name', + last_modified_field: '', + vector_field: 'text_vector', + select_content: true + }, + custom: { + content_field: 'content', + title_field: 'metadata_storage_name', + path_field: 'metadata_storage_path', + name_field: 'metadata_storage_name', + last_modified_field: 'metadata_storage_last_modified', + vector_field: '', + select_content: false + } +}; +const AZURE_FILES_INDEX_LAYOUT_HELP = { + document_per_file: 'One search document represents each file. Uses metadata_storage_* fields from the Azure Files indexer.', + chunked: 'The index stores chunks with integrated vectorization and selects chunk content for snippets.', + custom: 'Use custom field names when the index was transformed or projected differently.' +}; +const AZURE_FILES_INDEX_RESOURCE_ID_PATTERN = /^\/subscriptions\/[0-9a-fA-F-]{36}\/resourceGroups\/[^/]+\/providers\/Microsoft\.Storage\/storageAccounts\/[A-Za-z0-9]{3,24}$/; +const AZURE_FILES_INDEX_NAME_PATTERN = /^[a-z0-9](?:[a-z0-9]|-(?=[a-z0-9])){1,127}$/; +const AZURE_FILES_INDEX_SHARE_NAME_PATTERN = /^[a-z0-9](?:[a-z0-9]|-(?=[a-z0-9])){2,62}$/; +const AZURE_FILES_INDEX_SEARCH_ENDPOINT_PATTERN = /^https:\/\/[A-Za-z0-9-]+\.search\.(windows\.net|azure\.us|azure\.cn)\/?$/; const ACTION_CONNECTION_TEST_CONFIG = { openapi: { idPrefix: 'openapi', url: '/api/plugins/test-openapi-connection', label: 'OpenAPI' }, azureMaps: { idPrefix: 'azure-maps', url: '/api/plugins/test-azure-maps-connection', label: 'Azure Maps' }, + azureFilesIndex: { idPrefix: 'azure-files-index', url: '/api/plugins/test-azure-files-index-connection', label: 'Azure Files Search' }, blobStorage: { idPrefix: 'blob-storage', url: '/api/plugins/test-blob-storage-connection', label: 'Blob Storage' }, databricks: { idPrefix: 'databricks', url: '/api/plugins/test-databricks-connection', label: 'Databricks' }, logAnalytics: { idPrefix: 'log-analytics', url: '/api/plugins/test-log-analytics-connection', label: 'Log Analytics' }, @@ -807,6 +847,32 @@ export class PluginModalStepper { document.getElementById('sql-identity-select').addEventListener('change', () => this.handleActionIdentityChange('sql')); document.getElementById('cosmos-auth-type').addEventListener('change', () => this.handleCosmosAuthTypeChange()); + const azureFilesLayoutSelect = document.getElementById('azure-files-index-layout'); + if (azureFilesLayoutSelect) { + azureFilesLayoutSelect.addEventListener('change', () => this.handleAzureFilesIndexLayoutChange()); + } + const azureFilesAuthTypeSelect = document.getElementById('azure-files-index-auth-type'); + if (azureFilesAuthTypeSelect) { + azureFilesAuthTypeSelect.addEventListener('change', () => this.handleAzureFilesIndexAuthTypeChange()); + } + const azureFilesPermissionModeSelect = document.getElementById('azure-files-index-permission-mode'); + if (azureFilesPermissionModeSelect) { + azureFilesPermissionModeSelect.addEventListener('change', () => this.handleAzureFilesIndexPermissionModeChange()); + } + const azureFilesAddShareButton = document.getElementById('azure-files-index-add-share-btn'); + if (azureFilesAddShareButton) { + azureFilesAddShareButton.addEventListener('click', () => this.addAzureFilesIndexShareRow()); + } + const azureFilesSharesList = document.getElementById('azure-files-index-shares-list'); + if (azureFilesSharesList) { + azureFilesSharesList.addEventListener('click', (event) => { + const removeButton = event.target.closest('.azure-files-index-remove-share-btn'); + if (removeButton) { + this.removeAzureFilesIndexShareRow(removeButton); + } + }); + } + const blobStorageAuthTypeSelect = document.getElementById('blob-storage-auth-type'); if (blobStorageAuthTypeSelect) { blobStorageAuthTypeSelect.addEventListener('change', () => this.handleBlobStorageAuthTypeChange()); @@ -1406,6 +1472,10 @@ export class PluginModalStepper { return !!(type && type.toLowerCase() === 'cosmos_query'); } + isAzureFilesIndexType(type = this.selectedType) { + return !!(type && type.toLowerCase() === AZURE_FILES_INDEX_PLUGIN_TYPE); + } + isRocksDbType(type = this.selectedType) { return !!(type && type.toLowerCase() === ROCKSDB_PLUGIN_TYPE); } @@ -3268,6 +3338,439 @@ export class PluginModalStepper { return Number.isNaN(value) ? defaultValue : value; } + setAzureFilesIndexFieldValue(fieldId, value = '') { + const field = document.getElementById(fieldId); + if (field) { + field.value = value === null || value === undefined ? '' : String(value); + } + } + + setAzureFilesIndexSummaryText(elementId, value) { + const element = document.getElementById(elementId); + if (element) { + element.textContent = value; + } + } + + getAzureFilesIndexLayoutDefaults(layout) { + return AZURE_FILES_INDEX_FIELD_DEFAULTS[layout] || AZURE_FILES_INDEX_FIELD_DEFAULTS.document_per_file; + } + + initializeAzureFilesIndexConfiguration() { + if (!document.getElementById('azure-files-index-config-section')) { + return; + } + + const defaults = [ + ['azure-files-index-layout', 'document_per_file'], + ['azure-files-index-query-mode', 'keyword'], + ['azure-files-index-auth-type', 'identity'], + ['azure-files-index-permission-mode', 'live_acl'], + ['azure-files-index-share-access-check', 'rbac'], + ['azure-files-index-default-top-n', '5'], + ['azure-files-index-max-candidates', '50'], + ['azure-files-index-max-snippet-chars', '1200'], + ['azure-files-index-time-budget', '25'] + ]; + + defaults.forEach(([fieldId, defaultValue]) => { + const field = document.getElementById(fieldId); + if (field && !String(field.value || '').trim()) { + field.value = defaultValue; + } + }); + + this.applyAzureFilesIndexLayoutDefaults(false); + this.ensureAzureFilesIndexShareRow(); + this.handleAzureFilesIndexAuthTypeChange(); + this.handleAzureFilesIndexPermissionModeChange(); + this.clearActionConnectionTestResult('azureFilesIndex'); + } + + handleAzureFilesIndexLayoutChange() { + this.applyAzureFilesIndexLayoutDefaults(true); + } + + applyAzureFilesIndexLayoutDefaults(force = false) { + const layout = document.getElementById('azure-files-index-layout')?.value || 'document_per_file'; + const defaults = this.getAzureFilesIndexLayoutDefaults(layout); + const fieldMap = { + content_field: 'azure-files-index-content-field', + title_field: 'azure-files-index-title-field', + path_field: 'azure-files-index-path-field', + name_field: 'azure-files-index-name-field', + last_modified_field: 'azure-files-index-last-modified-field', + vector_field: 'azure-files-index-vector-field' + }; + + Object.entries(fieldMap).forEach(([key, fieldId]) => { + const field = document.getElementById(fieldId); + if (field && (force || !String(field.value || '').trim())) { + field.value = defaults[key] || ''; + } + }); + + const selectContent = document.getElementById('azure-files-index-select-content'); + if (selectContent && (force || !selectContent.dataset.userTouched)) { + selectContent.checked = defaults.select_content === true; + } + if (selectContent && !selectContent.dataset.touchHandlerAttached) { + selectContent.addEventListener('change', () => { + selectContent.dataset.userTouched = 'true'; + }); + selectContent.dataset.touchHandlerAttached = 'true'; + } + + const help = document.getElementById('azure-files-index-layout-help'); + if (help) { + help.textContent = AZURE_FILES_INDEX_LAYOUT_HELP[layout] || AZURE_FILES_INDEX_LAYOUT_HELP.document_per_file; + } + } + + handleAzureFilesIndexAuthTypeChange() { + const authType = document.getElementById('azure-files-index-auth-type')?.value || 'identity'; + const keyGroup = document.getElementById('azure-files-index-auth-key-group'); + const keyInput = document.getElementById('azure-files-index-auth-key'); + keyGroup?.classList.toggle('d-none', authType !== 'key'); + if (keyInput) { + keyInput.required = authType === 'key'; + } + } + + handleAzureFilesIndexPermissionModeChange() { + const permissionMode = document.getElementById('azure-files-index-permission-mode')?.value || 'live_acl'; + const warning = document.getElementById('azure-files-index-permission-none-warning'); + warning?.classList.toggle('d-none', permissionMode !== 'none'); + } + + ensureAzureFilesIndexShareRow() { + const list = document.getElementById('azure-files-index-shares-list'); + if (list && list.children.length === 0) { + this.addAzureFilesIndexShareRow(); + } + } + + addAzureFilesIndexShareRow(storageAccountResourceId = '', shareName = '') { + const list = document.getElementById('azure-files-index-shares-list'); + if (!list) { + return; + } + + const row = document.createElement('div'); + row.className = 'azure-files-index-share-row'; + + const grid = document.createElement('div'); + grid.className = 'row g-2 align-items-end'; + + const resourceColumn = document.createElement('div'); + resourceColumn.className = 'col-md-7'; + const resourceLabel = document.createElement('label'); + resourceLabel.className = 'form-label'; + resourceLabel.textContent = 'Storage Account Resource ID'; + const resourceInput = document.createElement('input'); + resourceInput.type = 'text'; + resourceInput.className = 'form-control azure-files-index-share-resource-id'; + resourceInput.placeholder = '/subscriptions/.../resourceGroups/.../providers/Microsoft.Storage/storageAccounts/account'; + resourceInput.value = storageAccountResourceId || ''; + resourceColumn.append(resourceLabel, resourceInput); + + const shareColumn = document.createElement('div'); + shareColumn.className = 'col-md-3'; + const shareLabel = document.createElement('label'); + shareLabel.className = 'form-label'; + shareLabel.textContent = 'Share Name'; + const shareInput = document.createElement('input'); + shareInput.type = 'text'; + shareInput.className = 'form-control azure-files-index-share-name'; + shareInput.placeholder = 'department-files'; + shareInput.value = shareName || ''; + shareColumn.append(shareLabel, shareInput); + + const actionColumn = document.createElement('div'); + actionColumn.className = 'col-md-2 d-grid'; + const removeButton = document.createElement('button'); + removeButton.type = 'button'; + removeButton.className = 'btn btn-outline-danger azure-files-index-remove-share-btn'; + removeButton.append(document.createTextNode('Remove')); + actionColumn.appendChild(removeButton); + + grid.append(resourceColumn, shareColumn, actionColumn); + row.appendChild(grid); + list.appendChild(row); + } + + removeAzureFilesIndexShareRow(removeButton) { + removeButton.closest('.azure-files-index-share-row')?.remove(); + } + + getAzureFilesIndexStorageShares() { + return Array.from(document.querySelectorAll('#azure-files-index-shares-list .azure-files-index-share-row')) + .map(row => ({ + storage_account_resource_id: row.querySelector('.azure-files-index-share-resource-id')?.value.trim() || '', + share_name: row.querySelector('.azure-files-index-share-name')?.value.trim().toLowerCase() || '' + })) + .filter(share => share.storage_account_resource_id || share.share_name); + } + + getAzureFilesIndexConfiguration() { + const endpoint = (document.getElementById('azure-files-index-endpoint')?.value || '').trim().replace(/\/+$/, ''); + const authType = document.getElementById('azure-files-index-auth-type')?.value || 'identity'; + const auth = { type: authType }; + const additionalFields = { + index_name: (document.getElementById('azure-files-index-name')?.value || '').trim().toLowerCase(), + index_layout: document.getElementById('azure-files-index-layout')?.value || 'document_per_file', + content_field: (document.getElementById('azure-files-index-content-field')?.value || '').trim(), + title_field: (document.getElementById('azure-files-index-title-field')?.value || '').trim(), + path_field: (document.getElementById('azure-files-index-path-field')?.value || '').trim(), + name_field: (document.getElementById('azure-files-index-name-field')?.value || '').trim(), + last_modified_field: (document.getElementById('azure-files-index-last-modified-field')?.value || '').trim(), + vector_field: (document.getElementById('azure-files-index-vector-field')?.value || '').trim(), + select_content: document.getElementById('azure-files-index-select-content')?.checked === true, + query_mode: document.getElementById('azure-files-index-query-mode')?.value || 'keyword', + semantic_configuration: (document.getElementById('azure-files-index-semantic-configuration')?.value || '').trim(), + default_top_n: this.getIntegerFieldValue('azure-files-index-default-top-n', 5), + max_candidates: this.getIntegerFieldValue('azure-files-index-max-candidates', 50), + max_snippet_chars: this.getIntegerFieldValue('azure-files-index-max-snippet-chars', 1200), + time_budget_seconds: this.getIntegerFieldValue('azure-files-index-time-budget', 25), + permission_mode: document.getElementById('azure-files-index-permission-mode')?.value || 'live_acl', + permission_mode_none_acknowledged: document.getElementById('azure-files-index-permission-none-ack')?.checked === true, + share_access_check: document.getElementById('azure-files-index-share-access-check')?.value || 'rbac', + treat_builtin_users_as_member: document.getElementById('azure-files-index-builtin-users')?.checked === true, + storage_shares: this.getAzureFilesIndexStorageShares() + }; + + if (authType === 'key') { + const key = (document.getElementById('azure-files-index-auth-key')?.value || '').trim(); + if (key) { + auth.key = key; + } else if (this.isEditMode && this.originalPlugin?.auth?.key) { + auth.key = this.originalPlugin.auth.key; + } + } else { + auth.identity = 'managed_identity'; + } + + return { endpoint, auth, additionalFields }; + } + + getAzureFilesIndexValidationError() { + const config = this.getAzureFilesIndexConfiguration(); + const fields = config.additionalFields; + + if (!AZURE_FILES_INDEX_SEARCH_ENDPOINT_PATTERN.test(config.endpoint)) { + return 'Search service endpoint must be an Azure AI Search URL, for example https://service.search.windows.net.'; + } + if (!AZURE_FILES_INDEX_NAME_PATTERN.test(fields.index_name)) { + return 'Index name must be 2-128 lowercase letters, numbers, or single dashes.'; + } + if (config.auth.type === 'key' && !config.auth.key && !this.isEditMode) { + return 'Query key is required when using key authentication.'; + } + if (!fields.content_field) { + return 'Content field is required.'; + } + if (!fields.path_field) { + return 'Path field is required.'; + } + if (!fields.name_field) { + return 'Name field is required.'; + } + if (fields.query_mode === 'hybrid' && !fields.vector_field) { + return 'Hybrid search requires a vector field.'; + } + if (fields.default_top_n < 1 || fields.default_top_n > 20) { + return 'Default results must be between 1 and 20.'; + } + if (fields.max_candidates < 5 || fields.max_candidates > 200) { + return 'Max candidates must be between 5 and 200.'; + } + if (fields.max_snippet_chars < 200 || fields.max_snippet_chars > 4000) { + return 'Snippet characters must be between 200 and 4000.'; + } + if (fields.time_budget_seconds < 5 || fields.time_budget_seconds > 60) { + return 'Permission budget must be between 5 and 60 seconds.'; + } + if (fields.permission_mode === 'none' && !fields.permission_mode_none_acknowledged) { + return 'Acknowledge that every action user can search every file before disabling permission checks.'; + } + if (fields.permission_mode === 'live_acl' && fields.storage_shares.length === 0) { + return 'Add at least one Azure file share for live ACL permission checks.'; + } + + const invalidShare = fields.storage_shares.find(share => ( + !AZURE_FILES_INDEX_RESOURCE_ID_PATTERN.test(share.storage_account_resource_id) + || !AZURE_FILES_INDEX_SHARE_NAME_PATTERN.test(share.share_name) + )); + if (invalidShare) { + return 'Each storage share needs a valid storage account resource ID and a 3-63 character lowercase share name.'; + } + + return null; + } + + populateAzureFilesIndexForm(plugin) { + const additionalFields = plugin.additionalFields || plugin.additional_fields || {}; + const auth = plugin.auth || {}; + + this.setAzureFilesIndexFieldValue('azure-files-index-endpoint', plugin.endpoint || ''); + this.setAzureFilesIndexFieldValue('azure-files-index-name', additionalFields.index_name || ''); + this.setAzureFilesIndexFieldValue('azure-files-index-layout', additionalFields.index_layout || 'document_per_file'); + this.applyAzureFilesIndexLayoutDefaults(true); + + // Blank field names fall back to the layout defaults on the server, so keep the defaults visible. + [ + ['azure-files-index-content-field', additionalFields.content_field], + ['azure-files-index-title-field', additionalFields.title_field], + ['azure-files-index-path-field', additionalFields.path_field], + ['azure-files-index-name-field', additionalFields.name_field], + ['azure-files-index-last-modified-field', additionalFields.last_modified_field], + ['azure-files-index-vector-field', additionalFields.vector_field] + ].forEach(([fieldId, value]) => { + if (value !== undefined && value !== null && String(value).trim()) { + this.setAzureFilesIndexFieldValue(fieldId, value); + } + }); + + [ + ['azure-files-index-query-mode', additionalFields.query_mode || 'keyword'], + ['azure-files-index-semantic-configuration', additionalFields.semantic_configuration || ''], + ['azure-files-index-permission-mode', additionalFields.permission_mode || 'live_acl'], + ['azure-files-index-share-access-check', additionalFields.share_access_check || 'rbac'], + ['azure-files-index-default-top-n', additionalFields.default_top_n || 5], + ['azure-files-index-max-candidates', additionalFields.max_candidates || 50], + ['azure-files-index-max-snippet-chars', additionalFields.max_snippet_chars || 1200], + ['azure-files-index-time-budget', additionalFields.time_budget_seconds || 25] + ].forEach(([fieldId, value]) => this.setAzureFilesIndexFieldValue(fieldId, value)); + + const selectContent = document.getElementById('azure-files-index-select-content'); + if (selectContent) { + if (Object.prototype.hasOwnProperty.call(additionalFields, 'select_content')) { + selectContent.checked = additionalFields.select_content === true; + selectContent.dataset.userTouched = 'true'; + } else { + delete selectContent.dataset.userTouched; + } + } + const noneAck = document.getElementById('azure-files-index-permission-none-ack'); + if (noneAck) { + noneAck.checked = additionalFields.permission_mode_none_acknowledged === true; + } + const builtinUsers = document.getElementById('azure-files-index-builtin-users'); + if (builtinUsers) { + builtinUsers.checked = additionalFields.treat_builtin_users_as_member === true; + } + + this.setAzureFilesIndexFieldValue('azure-files-index-auth-type', auth.type === 'key' ? 'key' : 'identity'); + this.setAzureFilesIndexFieldValue('azure-files-index-auth-key', auth.type === 'key' ? (auth.key || '') : ''); + + const sharesList = document.getElementById('azure-files-index-shares-list'); + if (sharesList) { + sharesList.replaceChildren(); + const shares = Array.isArray(additionalFields.storage_shares) ? additionalFields.storage_shares : []; + shares.forEach(share => this.addAzureFilesIndexShareRow( + share.storage_account_resource_id || '', + share.share_name || '' + )); + } + this.ensureAzureFilesIndexShareRow(); + this.handleAzureFilesIndexAuthTypeChange(); + this.handleAzureFilesIndexPermissionModeChange(); + } + + formatAzureFilesIndexLayout(layout) { + const labels = { + document_per_file: 'Document per file', + chunked: 'Chunked with vectors', + custom: 'Custom field mapping' + }; + return labels[layout] || layout || '-'; + } + + formatAzureFilesIndexQueryMode(mode) { + const labels = { + keyword: 'Keyword', + semantic: 'Semantic', + hybrid: 'Hybrid' + }; + return labels[mode] || mode || '-'; + } + + formatAzureFilesIndexPermissionMode(mode) { + return mode === 'none' ? 'No permission filtering' : 'Live NTFS ACL checks'; + } + + populateAzureFilesIndexSummary() { + const section = document.getElementById('summary-azure-files-index-section'); + if (!section) { + return; + } + + if (!this.isAzureFilesIndexType()) { + section.classList.add('d-none'); + return; + } + + const config = this.getAzureFilesIndexConfiguration(); + const fields = config.additionalFields; + this.setAzureFilesIndexSummaryText('summary-azure-files-index-name', fields.index_name || '-'); + this.setAzureFilesIndexSummaryText('summary-azure-files-index-layout', this.formatAzureFilesIndexLayout(fields.index_layout)); + this.setAzureFilesIndexSummaryText('summary-azure-files-index-query-mode', this.formatAzureFilesIndexQueryMode(fields.query_mode)); + this.setAzureFilesIndexSummaryText('summary-azure-files-index-permission-mode', this.formatAzureFilesIndexPermissionMode(fields.permission_mode)); + this.setAzureFilesIndexSummaryText( + 'summary-azure-files-index-shares', + fields.storage_shares.length ? `${fields.storage_shares.length} configured` : 'None configured' + ); + this.setAzureFilesIndexSummaryText( + 'summary-azure-files-index-limits', + `${fields.default_top_n} results, ${fields.max_candidates} candidates, ${fields.time_budget_seconds}s ACL budget` + ); + this.setAzureFilesIndexSummaryText( + 'summary-azure-files-index-fields', + `content=${fields.content_field || '-'}, path=${fields.path_field || '-'}, name=${fields.name_field || '-'}, vector=${fields.vector_field || 'none'}` + ); + section.classList.remove('d-none'); + } + + renderAzureFilesConnectionTestDetails(payload = {}) { + const elements = this.getActionConnectionTestElements('azureFilesIndex'); + if (!elements?.alert) { + return; + } + + const checks = Array.isArray(payload.details?.checks) ? payload.details.checks : []; + if (checks.length) { + const list = document.createElement('ul'); + list.className = 'list-unstyled mt-2 mb-0'; + checks.forEach(check => { + const item = document.createElement('li'); + item.className = 'd-flex align-items-start gap-2 mb-1'; + const badge = document.createElement('span'); + const status = String(check.status || '').toLowerCase(); + badge.className = status === 'pass' + ? 'badge text-bg-success' + : (status === 'warn' ? 'badge text-bg-warning' : 'badge text-bg-danger'); + badge.textContent = check.status || 'check'; + const text = document.createElement('span'); + text.textContent = `${check.name || 'Check'}: ${check.message || ''}`; + item.append(badge, text); + list.appendChild(item); + }); + elements.alert.appendChild(list); + } + + const warnings = Array.isArray(payload.warnings) ? payload.warnings : []; + if (warnings.length) { + const warningList = document.createElement('ul'); + warningList.className = 'mt-2 mb-0'; + warnings.forEach(warning => { + const item = document.createElement('li'); + item.textContent = warning; + warningList.appendChild(item); + }); + elements.alert.appendChild(warningList); + } + } + formatMcpTransport(transport) { const labels = { streamable_http: 'Streamable HTTP', @@ -4034,7 +4537,7 @@ export class PluginModalStepper { } isStructuredConfigType(type = this.selectedType) { - return this.isSqlType(type) || this.isCosmosType(type) || this.isRocksDbType(type) || this.isDocumentSearchType(type) || this.isBlobStorageType(type) || this.isDatabricksType(type) || this.isSnowflakeType(type) || this.isTableauType(type) || this.isYamcsType(type) || this.isMcpType(type) || this.isSimpleChatType(type) || this.isMsGraphType(type) || this.isAzureMapsType(type) || this.isChartType(type) || this.isLogAnalyticsType(type) || this.isAgentType(type); + return this.isSqlType(type) || this.isCosmosType(type) || this.isAzureFilesIndexType(type) || this.isRocksDbType(type) || this.isDocumentSearchType(type) || this.isBlobStorageType(type) || this.isDatabricksType(type) || this.isSnowflakeType(type) || this.isTableauType(type) || this.isYamcsType(type) || this.isMcpType(type) || this.isSimpleChatType(type) || this.isMsGraphType(type) || this.isAzureMapsType(type) || this.isChartType(type) || this.isLogAnalyticsType(type) || this.isAgentType(type); } showConfigSectionForType() { @@ -4043,6 +4546,7 @@ export class PluginModalStepper { generic: document.getElementById('generic-config-section'), sql: document.getElementById('sql-config-section'), cosmos: document.getElementById('cosmos-config-section'), + azureFilesIndex: document.getElementById('azure-files-index-config-section'), rocksdb: document.getElementById('rocksdb-config-section'), documentSearch: document.getElementById('document-search-config-section'), blobStorage: document.getElementById('blob-storage-config-section'), @@ -4076,6 +4580,9 @@ export class PluginModalStepper { } else if (this.isCosmosType()) { showOnly('cosmos'); this.initializeCosmosConfiguration(); + } else if (this.isAzureFilesIndexType()) { + showOnly('azureFilesIndex'); + this.initializeAzureFilesIndexConfiguration(); } else if (this.isRocksDbType()) { showOnly('rocksdb'); this.initializeRocksDbConfiguration(); @@ -4147,6 +4654,7 @@ export class PluginModalStepper { const isOpenApiType = this.isOpenApiType(); const isSqlType = this.isSqlType(); const isCosmosType = this.isCosmosType(); + const isAzureFilesIndexType = this.isAzureFilesIndexType(); const isDocumentSearchType = this.isDocumentSearchType(); const isBlobStorageType = this.isBlobStorageType(); const isDatabricksType = this.isDatabricksType(); @@ -4162,6 +4670,8 @@ export class PluginModalStepper { titleEl.textContent = 'Database Configuration'; } else if (isCosmosType) { titleEl.textContent = 'Cosmos Configuration'; + } else if (isAzureFilesIndexType) { + titleEl.textContent = 'Azure Files Search Configuration'; } else if (this.isRocksDbType()) { titleEl.textContent = 'RocksDB Configuration'; } else if (isDocumentSearchType) { @@ -4369,6 +4879,7 @@ export class PluginModalStepper { const openApiSection = document.getElementById('openapi-config-section'); const sqlSection = document.getElementById('sql-config-section'); const cosmosSection = document.getElementById('cosmos-config-section'); + const azureFilesIndexSection = document.getElementById('azure-files-index-config-section'); const rocksDbSection = document.getElementById('rocksdb-config-section'); const documentSearchSection = document.getElementById('document-search-config-section'); const blobStorageSection = document.getElementById('blob-storage-config-section'); @@ -4386,6 +4897,7 @@ export class PluginModalStepper { const isOpenApiVisible = !openApiSection.classList.contains('d-none'); const isSqlVisible = !sqlSection.classList.contains('d-none'); const isCosmosVisible = !cosmosSection.classList.contains('d-none'); + const isAzureFilesIndexVisible = !!azureFilesIndexSection && !azureFilesIndexSection.classList.contains('d-none'); const isRocksDbVisible = !!rocksDbSection && !rocksDbSection.classList.contains('d-none'); const isDocumentSearchVisible = !documentSearchSection.classList.contains('d-none'); const isBlobStorageVisible = !blobStorageSection.classList.contains('d-none'); @@ -4549,6 +5061,12 @@ export class PluginModalStepper { this.showError('Timeout must be between 1 and 120 seconds.'); return false; } + } else if (isAzureFilesIndexVisible) { + const azureFilesIndexError = this.getAzureFilesIndexValidationError(); + if (azureFilesIndexError) { + this.showError(azureFilesIndexError); + return false; + } } else if (isRocksDbVisible) { const rocksDbError = this.getRocksDbValidationError(); if (rocksDbError) { @@ -6159,6 +6677,14 @@ export class PluginModalStepper { return azureMapsConfig; } + if (testKey === 'azureFilesIndex') { + const azureFilesIndexError = this.getAzureFilesIndexValidationError(); + if (azureFilesIndexError) { + throw new Error(azureFilesIndexError); + } + return this.getAzureFilesIndexConfiguration(); + } + if (testKey === 'blobStorage') { const blobStorageConfig = this.getBlobStorageConfiguration(); if (!blobStorageConfig.additionalFields.container_name) { @@ -6291,6 +6817,9 @@ export class PluginModalStepper { data.message || 'Connection successful.', 'bi bi-check-circle' ); + if (testKey === 'azureFilesIndex') { + this.renderAzureFilesConnectionTestDetails(data); + } } else { this.showActionConnectionTestMessage( testKey, @@ -6298,6 +6827,9 @@ export class PluginModalStepper { data.error || `The ${elements.config.label} connection test failed.`, 'bi bi-x-circle' ); + if (testKey === 'azureFilesIndex') { + this.renderAzureFilesConnectionTestDetails(data); + } } } catch (error) { this.showActionConnectionTestMessage( @@ -6871,6 +7403,8 @@ export class PluginModalStepper { document.getElementById('cosmos-auth-type').value = auth.type || 'identity'; document.getElementById('cosmos-auth-key').value = auth.key || ''; this.initializeCosmosConfiguration(); + } else if (this.isAzureFilesIndexType(plugin.type)) { + this.populateAzureFilesIndexForm(plugin); } else if (this.isRocksDbType(plugin.type)) { this.populateRocksDbForm(plugin); } else if (this.isDocumentSearchType(plugin.type)) { @@ -6981,6 +7515,7 @@ export class PluginModalStepper { const openApiSection = document.getElementById('openapi-config-section'); const sqlSection = document.getElementById('sql-config-section'); const cosmosSection = document.getElementById('cosmos-config-section'); + const azureFilesIndexSection = document.getElementById('azure-files-index-config-section'); const rocksDbSection = document.getElementById('rocksdb-config-section'); const documentSearchSection = document.getElementById('document-search-config-section'); const blobStorageSection = document.getElementById('blob-storage-config-section'); @@ -6994,6 +7529,7 @@ export class PluginModalStepper { const isOpenApiVisible = !openApiSection.classList.contains('d-none'); const isSqlVisible = !sqlSection.classList.contains('d-none'); const isCosmosVisible = !cosmosSection.classList.contains('d-none'); + const isAzureFilesIndexVisible = !!azureFilesIndexSection && !azureFilesIndexSection.classList.contains('d-none'); const isRocksDbVisible = !!rocksDbSection && !rocksDbSection.classList.contains('d-none'); const isDocumentSearchVisible = !documentSearchSection.classList.contains('d-none'); const isBlobStorageVisible = !blobStorageSection.classList.contains('d-none'); @@ -7192,6 +7728,16 @@ export class PluginModalStepper { additionalFields.field_hints = this.getCosmosFieldHints(); additionalFields.max_items = parseInt(document.getElementById('cosmos-max-items').value, 10) || 100; additionalFields.timeout = parseInt(document.getElementById('cosmos-timeout').value, 10) || 30; + } else if (isAzureFilesIndexVisible) { + const azureFilesIndexError = this.getAzureFilesIndexValidationError(); + if (azureFilesIndexError) { + throw new Error(azureFilesIndexError); + } + + const azureFilesIndexConfig = this.getAzureFilesIndexConfiguration(); + endpoint = azureFilesIndexConfig.endpoint; + auth = azureFilesIndexConfig.auth; + additionalFields = azureFilesIndexConfig.additionalFields; } else if (isRocksDbVisible) { const rocksDbError = this.getRocksDbValidationError(); if (rocksDbError) { @@ -7364,6 +7910,7 @@ export class PluginModalStepper { // Configuration Section - Handle endpoint vs SQL/Cosmos configuration const isSqlType = this.isSqlType(); const isCosmosType = this.isCosmosType(); + const isAzureFilesIndexType = this.isAzureFilesIndexType(); const isDocumentSearchType = this.isDocumentSearchType(); const isBlobStorageType = this.isBlobStorageType(); const isDatabricksType = this.isDatabricksType(); @@ -7393,6 +7940,12 @@ export class PluginModalStepper { endpointRow.style.display = ''; document.getElementById('summary-plugin-database-type').textContent = 'Cosmos DB for NoSQL'; databaseTypeRow.style.display = ''; + } else if (isAzureFilesIndexType) { + const endpoint = this.getEndpointValue(); + document.getElementById('summary-plugin-endpoint').textContent = endpoint || '-'; + endpointRow.style.display = ''; + document.getElementById('summary-plugin-database-type').textContent = 'Azure AI Search index over Azure Files'; + databaseTypeRow.style.display = ''; } else if (isRocksDbType) { const endpoint = this.getEndpointValue(); document.getElementById('summary-plugin-endpoint').textContent = endpoint || '-'; @@ -7481,10 +8034,10 @@ export class PluginModalStepper { } const databaseType = this.getSqlDatabaseType(); - if (!isSqlType && !isCosmosType && !isRocksDbType && !isDocumentSearchType && !isBlobStorageType && !isDatabricksType && !isSnowflakeType && !isTableauType && !isYamcsType && !isMcpType && !isSimpleChatType && !isMsGraphType && !isAzureMapsType && !isChartType && !isLogAnalyticsType && !isAgentType && databaseType) { + if (!isSqlType && !isCosmosType && !isAzureFilesIndexType && !isRocksDbType && !isDocumentSearchType && !isBlobStorageType && !isDatabricksType && !isSnowflakeType && !isTableauType && !isYamcsType && !isMcpType && !isSimpleChatType && !isMsGraphType && !isAzureMapsType && !isChartType && !isLogAnalyticsType && !isAgentType && databaseType) { document.getElementById('summary-plugin-database-type').textContent = databaseType; databaseTypeRow.style.display = ''; - } else if (!isSqlType && !isCosmosType && !isRocksDbType && !isDocumentSearchType && !isBlobStorageType && !isDatabricksType && !isSnowflakeType && !isTableauType && !isYamcsType && !isMcpType && !isSimpleChatType && !isMsGraphType && !isAzureMapsType && !isChartType && !isLogAnalyticsType && !isAgentType) { + } else if (!isSqlType && !isCosmosType && !isAzureFilesIndexType && !isRocksDbType && !isDocumentSearchType && !isBlobStorageType && !isDatabricksType && !isSnowflakeType && !isTableauType && !isYamcsType && !isMcpType && !isSimpleChatType && !isMsGraphType && !isAzureMapsType && !isChartType && !isLogAnalyticsType && !isAgentType) { databaseTypeRow.style.display = 'none'; } @@ -7492,6 +8045,7 @@ export class PluginModalStepper { this.populateOpenApiSummary(); this.populateSqlSummary(); this.populateCosmosSummary(); + this.populateAzureFilesIndexSummary(); this.populateRocksDbSummary(); this.populateDocumentSearchSummary(); this.populateBlobStorageSummary(); @@ -7513,6 +8067,7 @@ export class PluginModalStepper { const isOpenApiType = this.isOpenApiType(); const isSqlType = this.isSqlType(); const isCosmosType = this.isCosmosType(); + const isAzureFilesIndexType = this.isAzureFilesIndexType(); const isDocumentSearchType = this.isDocumentSearchType(); const isBlobStorageType = this.isBlobStorageType(); const isDatabricksType = this.isDatabricksType(); @@ -7531,6 +8086,8 @@ export class PluginModalStepper { return document.getElementById('sql-connection-string').value.trim(); } else if (isCosmosType) { return document.getElementById('cosmos-endpoint').value.trim(); + } else if (isAzureFilesIndexType) { + return (document.getElementById('azure-files-index-endpoint')?.value || '').trim().replace(/\/+$/, ''); } else if (this.isRocksDbType()) { return this.getRocksDbEndpointValue(); } else if (isDocumentSearchType) { @@ -7571,6 +8128,7 @@ export class PluginModalStepper { const isOpenApiType = this.isOpenApiType(); const isSqlType = this.isSqlType(); const isCosmosType = this.isCosmosType(); + const isAzureFilesIndexType = this.isAzureFilesIndexType(); const isDocumentSearchType = this.isDocumentSearchType(); const isBlobStorageType = this.isBlobStorageType(); const isDatabricksType = this.isDatabricksType(); @@ -7592,6 +8150,9 @@ export class PluginModalStepper { } else if (isCosmosType) { const authType = document.getElementById('cosmos-auth-type')?.value || 'identity'; return authType === 'key' ? 'Account Key' : 'Managed Identity'; + } else if (isAzureFilesIndexType) { + const authType = document.getElementById('azure-files-index-auth-type')?.value || 'identity'; + return authType === 'key' ? 'Query Key' : 'Managed Identity'; } else if (this.isRocksDbType()) { return this.getRocksDbAuthLabel(); } else if (isDocumentSearchType) { @@ -8237,6 +8798,7 @@ export class PluginModalStepper { const isOpenApiType = this.isOpenApiType(); const isSqlType = this.isSqlType(); const isCosmosType = this.isCosmosType(); + const isAzureFilesIndexType = this.isAzureFilesIndexType(); const isDocumentSearchType = this.isDocumentSearchType(); const isDatabricksType = this.isDatabricksType(); const isTableauType = this.isTableauType(); @@ -8254,6 +8816,8 @@ export class PluginModalStepper { currentEndpoint = document.getElementById('sql-connection-string')?.value || ''; } else if (isCosmosType) { currentEndpoint = document.getElementById('cosmos-endpoint')?.value || ''; + } else if (isAzureFilesIndexType) { + currentEndpoint = this.getEndpointValue(); } else if (this.isRocksDbType()) { currentEndpoint = this.getRocksDbEndpointValue(); } else if (isDocumentSearchType) { @@ -8302,6 +8866,11 @@ export class PluginModalStepper { if (currentAuthType === 'key') { currentAuthKey = document.getElementById('cosmos-auth-key')?.value || ''; } + } else if (isAzureFilesIndexType) { + currentAuthType = document.getElementById('azure-files-index-auth-type')?.value || 'identity'; + if (currentAuthType === 'key') { + currentAuthKey = document.getElementById('azure-files-index-auth-key')?.value || ''; + } } else if (this.isRocksDbType()) { const rocksDbAuthScheme = document.getElementById('rocksdb-auth-scheme')?.value || ROCKSDB_AUTH_SCHEME_NONE; currentAuthType = rocksDbAuthScheme !== ROCKSDB_AUTH_SCHEME_NONE ? 'key' : 'NoAuth'; @@ -8372,6 +8941,8 @@ export class PluginModalStepper { max_items: parseInt(document.getElementById('cosmos-max-items')?.value, 10) || 100, timeout: parseInt(document.getElementById('cosmos-timeout')?.value, 10) || 30 }, null, 2); + } else if (isAzureFilesIndexType) { + currentAdditionalFields = JSON.stringify(this.getAzureFilesIndexConfiguration().additionalFields, null, 2); } else if (this.isRocksDbType()) { currentAdditionalFields = JSON.stringify(this.getRocksDbConfiguration().additionalFields, null, 2); } else if (isDocumentSearchType) { @@ -8788,6 +9359,44 @@ export class PluginModalStepper { safeSetValue('cosmos-auth-type', 'identity'); safeSetValue('cosmos-auth-key'); + // Step 3 fields - Azure Files Search Plugin + safeSetValue('azure-files-index-endpoint'); + safeSetValue('azure-files-index-name'); + safeSetValue('azure-files-index-auth-type', 'identity'); + safeSetValue('azure-files-index-auth-key'); + safeSetValue('azure-files-index-layout', 'document_per_file'); + safeSetValue('azure-files-index-query-mode', 'keyword'); + safeSetValue('azure-files-index-semantic-configuration'); + safeSetValue('azure-files-index-content-field', 'content'); + safeSetValue('azure-files-index-title-field', 'metadata_storage_name'); + safeSetValue('azure-files-index-path-field', 'metadata_storage_path'); + safeSetValue('azure-files-index-name-field', 'metadata_storage_name'); + safeSetValue('azure-files-index-last-modified-field', 'metadata_storage_last_modified'); + safeSetValue('azure-files-index-vector-field'); + safeSetValue('azure-files-index-permission-mode', 'live_acl'); + safeSetValue('azure-files-index-share-access-check', 'rbac'); + safeSetValue('azure-files-index-default-top-n', '5'); + safeSetValue('azure-files-index-max-candidates', '50'); + safeSetValue('azure-files-index-max-snippet-chars', '1200'); + safeSetValue('azure-files-index-time-budget', '25'); + const azureFilesSelectContent = document.getElementById('azure-files-index-select-content'); + if (azureFilesSelectContent) { + azureFilesSelectContent.checked = false; + delete azureFilesSelectContent.dataset.userTouched; + } + const azureFilesNoneAck = document.getElementById('azure-files-index-permission-none-ack'); + if (azureFilesNoneAck) { + azureFilesNoneAck.checked = false; + } + const azureFilesBuiltinUsers = document.getElementById('azure-files-index-builtin-users'); + if (azureFilesBuiltinUsers) { + azureFilesBuiltinUsers.checked = false; + } + const azureFilesSharesList = document.getElementById('azure-files-index-shares-list'); + if (azureFilesSharesList) { + azureFilesSharesList.replaceChildren(); + } + // Step 3 fields - Document Search Plugin safeSetValue('document-search-scope', 'all'); safeSetValue('document-search-top-n', '12'); @@ -8834,6 +9443,8 @@ export class PluginModalStepper { this.toggleGenericAuthFields(); this.handleSqlAuthTypeChange(); this.handleCosmosAuthTypeChange(); + this.handleAzureFilesIndexAuthTypeChange(); + this.handleAzureFilesIndexPermissionModeChange(); } catch (e) { console.log('Some auth field toggles not available:', e.message); } diff --git a/application/single_app/static/js/workspace/workspace-file-sync.js b/application/single_app/static/js/workspace/workspace-file-sync.js index aeb062ccc..3927fe9db 100644 --- a/application/single_app/static/js/workspace/workspace-file-sync.js +++ b/application/single_app/static/js/workspace/workspace-file-sync.js @@ -1054,6 +1054,11 @@ function initializeFileSyncRoot(root) { }); const azureBlobCredentialAssessment = createElement('div', { className: 'd-none' }); renderCredentialMetadata(azureBlobCredentialAssessment, source?.credential_metadata, false); + const azureFilesCredentialNotice = createElement('div', { + className: 'alert alert-info py-2 mb-3', + attributes: { 'data-file-sync-azure-files-notice': 'true' }, + text: 'A managed identity or service principal reads the share over Azure Files OAuth, so it needs the Storage File Data Privileged Reader role on the storage account or file share. That role reads every file regardless of NTFS permissions, and everyone in this workspace can search the synced files. A connection string or SAS also works.', + }); const localCredentialsWrapper = createElement('div', { className: 'row g-3' }); const authTypeWrapper = createElement('div', { className: 'col-md-6' }); @@ -1214,6 +1219,8 @@ function initializeFileSyncRoot(root) { identityAndAuthenticationChildren = [globalConnectorNotice]; } else if (selectedSourceType === 'azure_blob') { identityAndAuthenticationChildren = [azureBlobCredentialNotice, azureBlobCredentialAssessment, identityWrapper, localCredentialsWrapper]; + } else if (selectedSourceType === 'azure_files') { + identityAndAuthenticationChildren = [azureFilesCredentialNotice, identityWrapper, localCredentialsWrapper]; } appendChildren(content, [ diff --git a/application/single_app/static/json/schemas/azure_files_index.definition.json b/application/single_app/static/json/schemas/azure_files_index.definition.json new file mode 100644 index 000000000..317ef0da6 --- /dev/null +++ b/application/single_app/static/json/schemas/azure_files_index.definition.json @@ -0,0 +1,7 @@ +{ + "$schema": "./plugin.definition.schema.json", + "allowedAuthTypes": [ + "identity", + "key" + ] +} diff --git a/application/single_app/static/json/schemas/azure_files_index_plugin.additional_settings.schema.json b/application/single_app/static/json/schemas/azure_files_index_plugin.additional_settings.schema.json new file mode 100644 index 000000000..8ecb6c98d --- /dev/null +++ b/application/single_app/static/json/schemas/azure_files_index_plugin.additional_settings.schema.json @@ -0,0 +1,128 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Azure Files Search Additional Settings", + "type": "object", + "properties": { + "index_name": { + "type": "string", + "description": "Name of the Azure AI Search index built by the Azure Files indexer." + }, + "index_layout": { + "type": "string", + "enum": ["document_per_file", "chunked", "custom"], + "default": "document_per_file", + "description": "document_per_file for the indexer's default one-document-per-file index, chunked for integrated vectorization with index projections, or custom to map every field." + }, + "content_field": { + "type": "string", + "description": "Searchable text field. Defaults to content (document per file) or chunk (chunked)." + }, + "title_field": { + "type": "string", + "description": "Optional title field shown with results." + }, + "path_field": { + "type": "string", + "description": "Field holding the file's Azure Files URL. Defaults to metadata_storage_path." + }, + "name_field": { + "type": "string", + "description": "Field holding the file name. Defaults to metadata_storage_name." + }, + "last_modified_field": { + "type": "string", + "description": "Optional last-modified field." + }, + "vector_field": { + "type": "string", + "description": "Vector field used by hybrid search through the index's vectorizer." + }, + "select_content": { + "type": "boolean", + "description": "Return the content field for snippets. Chunked indexes do; whole-file indexes use captions or highlights instead." + }, + "query_mode": { + "type": "string", + "enum": ["keyword", "semantic", "hybrid"], + "default": "keyword", + "description": "keyword, semantic (needs a semantic configuration), or hybrid (needs a vector field with a vectorizer)." + }, + "semantic_configuration": { + "type": "string", + "description": "Semantic configuration name. Leave blank to use the index default." + }, + "default_top_n": { + "type": "integer", + "default": 5, + "minimum": 1, + "maximum": 20, + "description": "Results returned when the agent does not ask for a number." + }, + "max_candidates": { + "type": "integer", + "default": 50, + "minimum": 5, + "maximum": 200, + "description": "Search hits examined per query before permission checks." + }, + "max_snippet_chars": { + "type": "integer", + "default": 1200, + "minimum": 200, + "maximum": 4000, + "description": "Maximum characters of text returned per result." + }, + "time_budget_seconds": { + "type": "integer", + "default": 25, + "minimum": 5, + "maximum": 60, + "description": "Time allowed for permission checks; files not checked in time are withheld." + }, + "permission_mode": { + "type": "string", + "enum": ["live_acl", "none"], + "default": "live_acl", + "description": "live_acl checks each file's permissions for the signed-in user; none returns every matching file." + }, + "permission_mode_none_acknowledged": { + "type": "boolean", + "default": false, + "description": "Required when permission_mode is none." + }, + "share_access_check": { + "type": "string", + "enum": ["rbac", "skip"], + "default": "rbac", + "description": "rbac checks share-level permissions (default share permission and Azure role assignments); skip relies on file permissions only." + }, + "treat_builtin_users_as_member": { + "type": "boolean", + "default": false, + "description": "Treat entries for BUILTIN\\Users as applying to every signed-in user." + }, + "storage_shares": { + "type": "array", + "default": [], + "maxItems": 50, + "items": { + "type": "object", + "properties": { + "storage_account_resource_id": { + "type": "string", + "description": "Azure resource ID of the storage account." + }, + "share_name": { + "type": "string", + "description": "File share name." + } + }, + "required": ["storage_account_resource_id", "share_name"], + "additionalProperties": false + }, + "description": "File shares this index was built from. Results from other shares are withheld." + } + }, + "required": ["index_name"], + "additionalProperties": false +} diff --git a/application/single_app/templates/_plugin_modal.html b/application/single_app/templates/_plugin_modal.html index 731e988b1..5dc027732 100644 --- a/application/single_app/templates/_plugin_modal.html +++ b/application/single_app/templates/_plugin_modal.html @@ -1638,6 +1638,216 @@
{{ m365_label }} capabilities
+ +
+
+ Azure Files Search Configuration +
+ +
+
+ Search Index +
+ +
+ + +
Use the Azure AI Search service endpoint for the existing Azure Files index.
+
+ +
+
+ + +
2-128 lowercase letters, numbers, or single dashes.
+
+
+ + +
Managed identity is recommended for global actions.
+
+
+ +
+ + +
Leave unchanged when editing a Key Vault-backed key to preserve the existing secret.
+
+ +
+ + +
One search document represents each file. Uses metadata_storage_* fields from the Azure Files indexer.
+
+ +
+
+ + +
Hybrid search requires a vector field with an index vectorizer.
+
+
+ + +
Leave blank to use the index default, when available.
+
+
+ +
+
+
+ +
+
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+
+
+
+
+ +
+
+ File Permissions +
+ +
+ SimpleChat checks each file's NTFS permissions and share access for the signed-in user; files it can't verify are withheld and logged for admins. +
+ +
+
+ + +
+
+ + +
RBAC checks default share permissions and Azure role assignments; skip relies on file permissions only.
+
+
+ +
+
Permission filtering is disabled.
+
Everyone who can use this action can search every file in the index.
+
+ + +
+
+ +
+ + +
Keep this off unless the shares intentionally rely on broad Windows group access.
+
+ +
+
+
Allowed Azure file shares
+
Results from shares not listed here are withheld.
+
+ +
+
+
+ +
+
+ Limits +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
Seconds. Unchecked files are withheld.
+
+
+
+ +
+
Required Azure roles for the app identity
+
    +
  • Search Index Data Reader on the search service.
    • +
  • Storage File Data Privileged Reader and Reader on each storage account.
    • +
+
+ +
+
+ +
+
+ +
+
+
+
@@ -2101,6 +2311,61 @@
+ +
+
+
+ Azure Files Search Configuration +
+
+
+
+
+
+ + - +
+
+
+
+ + - +
+
+
+
+ + - +
+
+
+
+ + - +
+
+
+
+ + - +
+
+
+
+ + - +
+
+
+
+ + - +
+
+
+
+
+
@@ -2833,6 +3098,13 @@
background-color: #f8f9fa; } +#azure-files-index-config-section { + border: 1px solid #d7e7ff; + border-radius: 8px; + padding: 1rem; + background-color: #f8f9fa; +} + #rocksdb-config-section { border: 1px solid #ded4f2; border-radius: 8px; @@ -2866,6 +3138,21 @@
border: 1px solid #e9ecef; } +.azure-files-index-config-card { + background-color: white; + border-radius: 6px; + padding: 1rem; + margin-top: 1rem; + border: 1px solid #e9ecef; +} + +.azure-files-index-share-row { + background-color: #ffffff; + border: 1px solid #e9ecef; + border-radius: 6px; + padding: 0.75rem; +} + .rocksdb-config-card { background-color: white; border-radius: 6px; @@ -2959,6 +3246,10 @@
border-left: 4px solid #17a2b8; } +#azure-files-index-config-section .alert-info { + border-left: 1px solid #0dcaf0; +} + /* Plugin type selection styling */ .btn-outline-info small, .btn-outline-success small { @@ -3008,6 +3299,11 @@
background-color: #343a40; } +[data-bs-theme="dark"] #azure-files-index-config-section { + border-color: #495057; + background-color: #343a40; +} + [data-bs-theme="dark"] #rocksdb-config-section { border-color: #495057; background-color: #343a40; @@ -3017,6 +3313,8 @@
[data-bs-theme="dark"] .sql-auth-config, [data-bs-theme="dark"] .simplechat-config-card, [data-bs-theme="dark"] .cosmos-config-card, +[data-bs-theme="dark"] .azure-files-index-config-card, +[data-bs-theme="dark"] .azure-files-index-share-row, [data-bs-theme="dark"] .rocksdb-config-card, [data-bs-theme="dark"] .auth-section { background-color: #2d3237; diff --git a/application/single_app/templates/control_center.html b/application/single_app/templates/control_center.html index 95959d0f2..cf9d67f81 100644 --- a/application/single_app/templates/control_center.html +++ b/application/single_app/templates/control_center.html @@ -1352,6 +1352,7 @@
+ diff --git a/application/v2_ui/src/components/fileSources/FileSourceEditorDialog.tsx b/application/v2_ui/src/components/fileSources/FileSourceEditorDialog.tsx index af72f0125..9d59afdbb 100644 --- a/application/v2_ui/src/components/fileSources/FileSourceEditorDialog.tsx +++ b/application/v2_ui/src/components/fileSources/FileSourceEditorDialog.tsx @@ -392,6 +392,11 @@ export function FileSourceEditorDialog({
Authentication + {descriptor.notice ? ( +

+ {descriptor.notice} +

+ ) : null}