From b19478a6d05ee9840af5a272b3abec2d3f9ab228 Mon Sep 17 00:00:00 2001 From: Dhaval Shah Date: Fri, 22 May 2026 20:24:11 +0530 Subject: [PATCH] RANGER-5427 : AD Groups with 1500+ Users Fail to Sync into Ranger Admin via RangerUserSync --- .../util/UgsyncCommonConstants.java | 6 +- .../process/LdapUserGroupBuilder.java | 392 ++++++++++-------- .../config/UserGroupSyncConfig.java | 15 + .../process/PolicyMgrUserGroupBuilder.java | 3 +- .../process/TestLdapUserGroupBuilder.java | 272 +++++++++++- 5 files changed, 517 insertions(+), 171 deletions(-) diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java index 6f8905a3520..c86e2dba39a 100644 --- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java +++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java @@ -22,10 +22,14 @@ public class UgsyncCommonConstants { public enum CaseConversion { NONE, TO_LOWER, TO_UPPER } - public static final String ORIGINAL_NAME = "original_name"; + public static final String ORIGINAL_NAME = "original_name"; public static final String FULL_NAME = "full_name"; public static final String SYNC_SOURCE = "sync_source"; public static final String LDAP_URL = "ldap_url"; + public static final String PRINCIPAL = "ranger.usersync.kerberos.principal"; + public static final String AUTHENTICATION_TYPE = "hadoop.security.authentication"; + public static final String KEYTAB = "ranger.usersync.kerberos.keytab"; + public static final String NAME_RULE = "hadoop.security.auth_to_local"; public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none"; public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower"; diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java index 7f1463ce7d0..afcd7946b28 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java @@ -21,6 +21,7 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.security.SecureClientLogin; import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable; import org.apache.hadoop.thirdparty.com.google.common.collect.Table; import org.apache.ranger.ugsyncutil.model.LdapSyncSourceInfo; @@ -49,7 +50,11 @@ import javax.naming.ldap.Rdn; import javax.naming.ldap.StartTlsRequest; import javax.naming.ldap.StartTlsResponse; +import javax.security.auth.Subject; +import java.io.IOException; +import java.net.UnknownHostException; +import java.security.PrivilegedAction; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Arrays; @@ -70,6 +75,7 @@ public class LdapUserGroupBuilder implements UserGroupSource { private static final String DATE_FORMAT = "yyyyMMddHHmmss"; private static final String MEMBER_OF_ATTR = "memberof="; private static final String GROUP_NAME_ATTRIBUTE = "cn="; + public static String localHostname = "unknown"; private static final int PAGE_SIZE = 500; /* for AD uSNChanged */ @@ -98,6 +104,9 @@ public class LdapUserGroupBuilder implements UserGroupSource { private String ldapBindDn; private String ldapBindPassword; private String ldapAuthenticationMechanism; + private String principal; + private String keytab; + private String nameRules; private String ldapReferral; private String searchBase; private String userNameAttribute; @@ -280,7 +289,7 @@ public void updateSink(UserGroupSink sink) throws Throwable { } } - private void createLdapContext() throws Throwable { + private void initializeLdapContext() throws Throwable { Properties env = new Properties(); env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); @@ -318,7 +327,16 @@ private void createLdapContext() throws Throwable { } } - ldapContext = new InitialLdapContext(env, null); + env.put(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism); + env.put(Context.SECURITY_PRINCIPAL, ldapBindDn); + env.put(Context.SECURITY_CREDENTIALS, ldapBindPassword); + env.put(Context.REFERRAL, ldapReferral); + + try { + ldapContext = new InitialLdapContext(env, null); + } catch (NamingException e) { + throw e; + } if (!ldapUrl.startsWith("ldaps")) { if (config.isStartTlsEnabled()) { @@ -333,11 +351,22 @@ private void createLdapContext() throws Throwable { LOG.info("Starting TLS session..."); } } + } - ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapBindDn); - ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS, ldapBindPassword); - ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism); - ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral); + private void createLdapContext() throws Throwable { + if (ldapAuthenticationMechanism.equalsIgnoreCase("GSSAPI")) { + Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + Subject.doAs(sub, (PrivilegedAction) () -> { + try { + initializeLdapContext(); + } catch (Throwable e) { + LOG.error("Failed to create LDAP context: ", e); + } + return null; + }); + } else { + initializeLdapContext(); + } } private void setConfig() throws Throwable { @@ -351,6 +380,13 @@ private void setConfig() throws Throwable { ldapBindDn = config.getLdapBindDn(); ldapBindPassword = config.getLdapBindPassword(); ldapAuthenticationMechanism = config.getLdapAuthenticationMechanism(); + try { + principal = SecureClientLogin.getPrincipal(config.getProperty(UgsyncCommonConstants.PRINCIPAL, ""), localHostname); + } catch (IOException ignored) { + // do nothing + } + keytab = config.getProperty(UgsyncCommonConstants.KEYTAB, ""); + nameRules = config.getProperty(UgsyncCommonConstants.NAME_RULE, "DEFAULT"); ldapReferral = config.getContextReferral(); searchBase = config.getSearchBase(); userSearchBase = config.getUserSearchBase().split(";"); @@ -649,28 +685,7 @@ private long getUsers(boolean computeDeletes) throws Throwable { } } - // Examine the paged results control response - Control[] controls = ldapContext.getResponseControls(); - - if (controls != null) { - for (Control control : controls) { - if (control instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; - - total = prrc.getResultSize(); - - if (total != 0) { - LOG.debug("END-OF-PAGE total : {}", total); - } else { - LOG.debug("END-OF-PAGE total : unknown"); - } - - cookie = prrc.getCookie(); - } - } - } else { - LOG.debug("No controls were sent from the server"); - } + cookie = handlePagedResults(); // Re-activate paged results if (pagedResultsEnabled) { @@ -708,6 +723,7 @@ private long getGroups(boolean computeDeletes) throws Throwable { NamingEnumeration groupSearchResultEnum = null; DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); long highestdeltaSyncGroupTime = deltaSyncGroupTime; + Map> largeGroups = new HashMap<>(); try { createLdapContext(); @@ -741,15 +757,16 @@ private long getGroups(boolean computeDeletes) throws Throwable { LOG.info("extendedAllGroupsSearchFilter = {}", extendedAllGroupsSearchFilter); - for (String s : groupSearchBase) { + for (int ou = 0; ou < groupSearchBase.length; ou++) { byte[] cookie = null; int counter = 0; + Set largeGroupNames = new HashSet<>(); try { int paged = 0; do { - groupSearchResultEnum = ldapContext.search(s, extendedAllGroupsSearchFilter, groupSearchControls); + groupSearchResultEnum = ldapContext.search(groupSearchBase[ou], extendedAllGroupsSearchFilter, groupSearchControls); while (groupSearchResultEnum.hasMore()) { final SearchResult groupEntry = groupSearchResultEnum.next(); @@ -823,71 +840,22 @@ private long getGroups(boolean computeDeletes) throws Throwable { int userCount = 0; if (groupMemberAttr == null || groupMemberAttr.size() <= 0) { - try { - LOG.info("No members available for {}", gName); - } catch (Exception e) { - throw new RuntimeException(e); + if (config.isLargeGroupSyncEnabled()) { + groupMemberAttr = attributes.get("member;range=0-1499"); } - - sourceGroupUsers.put(groupFullName, new HashSet<>()); - continue; - } - - NamingEnumeration userEnum = groupMemberAttr.getAll(); - - while (userEnum.hasMore()) { - String originalUserFullName = (String) userEnum.next(); - - if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) { + if (groupMemberAttr == null || groupMemberAttr.size() <= 0) { + LOG.info("No members available for {}", gName); sourceGroupUsers.put(groupFullName, new HashSet<>()); continue; + } else { + largeGroupNames.add(gName); } - - userCount++; - - if (!userSearchEnabled) { - Map userAttrMap = new HashMap<>(); - String userName = getShortName(originalUserFullName); - - userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); - userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName); - userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); - userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); - - sourceUsers.put(originalUserFullName, userAttrMap); - - LOG.debug("As usersearch is disabled, adding user {} from group member attribute for group {}", userName, gName); - } - - groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName); } + userCount = processGroupMembers(groupMemberAttr, groupFullName); LOG.info("No. of members in the group {} = {}", gName, userCount); } - - // Examine the paged results control response - Control[] controls = ldapContext.getResponseControls(); - - if (controls != null) { - for (Control control : controls) { - if (control instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; - - total = prrc.getResultSize(); - - if (total != 0) { - LOG.debug("END-OF-PAGE total: {}", total); - } else { - LOG.debug("END-OF-PAGE total: unknown"); - } - - cookie = prrc.getCookie(); - } - } - } else { - LOG.debug("No controls were sent from the server"); - } - + cookie = handlePagedResults(); // Re-activate paged results if (pagedResultsEnabled) { LOG.debug("Fetched paged results round: {}", ++paged); @@ -901,6 +869,10 @@ private long getGroups(boolean computeDeletes) throws Throwable { LOG.error("LdapUserGroupBuilder.getGroups() failed with exception: ", t); LOG.info("LdapUserGroupBuilder.getGroups() group count: {}", counter); } + if (config.isLargeGroupSyncEnabled() && !largeGroupNames.isEmpty()) { + LOG.info("Large groups found: {}", largeGroupNames); + largeGroups.put(groupSearchBase[ou], largeGroupNames); + } } } finally { if (groupSearchResultEnum != null) { @@ -908,6 +880,10 @@ private long getGroups(boolean computeDeletes) throws Throwable { } closeLdapContext(); + if (config.isLargeGroupSyncEnabled() && MapUtils.isNotEmpty(largeGroups)) { + LOG.info("Large groups found: {}", largeGroups); + getRemainingGroupMembers(largeGroups); + } } if (groupHierarchyLevels > 0) { @@ -1056,59 +1032,10 @@ private void goUpGroupHierarchyLdap(Set groupDNs, int groupHierarchyLeve } sourceGroups.put(groupFullName, groupAttrMap); - - NamingEnumeration userEnum = groupMemberAttr.getAll(); - - while (userEnum.hasMore()) { - String originalUserFullName = (String) userEnum.next(); - - if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) { - continue; - } - - userCount++; - - if (!userSearchEnabled && !sourceGroups.containsKey(originalUserFullName)) { - Map userAttrMap = new HashMap<>(); - String userName = getShortName(originalUserFullName); - - userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); - userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName); - userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); - userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); - - sourceUsers.put(originalUserFullName, userAttrMap); - } - - groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName); - } - + userCount = processGroupMembers(groupMemberAttr, groupFullName); LOG.info("No. of members in the group {} = {}", gName, userCount); } - - // Examine the paged results control response - Control[] controls = ldapContext.getResponseControls(); - - if (controls != null) { - for (Control control : controls) { - if (control instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; - - total = prrc.getResultSize(); - - if (total != 0) { - LOG.debug("END-OF-PAGE total : {}", total); - } else { - LOG.debug("END-OF-PAGE total : unknown"); - } - - cookie = prrc.getCookie(); - } - } - } else { - LOG.debug("No controls were sent from the server"); - } - + cookie = handlePagedResults(); // Re-activate paged results if (pagedResultsEnabled) { ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)}); @@ -1149,7 +1076,7 @@ private void addToAttrMap(Map userAttrMap, String attrName, Attr } catch (ClassCastException e) { LOG.error("{} type is not set properly {}", attrName, e.getMessage()); } - } else if (attrType.equals("String")) { + } else if (attrType.equalsIgnoreCase("String")) { userAttrMap.put(attrName, (String) attr.get()); } else { LOG.warn("Attribute Type {} not supported for {}", attrType, attrName); @@ -1208,6 +1135,160 @@ private static String getShortName(String longName) { return shortName; } + private byte[] handlePagedResults() throws Throwable { + Control[] controls = ldapContext.getResponseControls(); + + if (controls != null) { + for (int i = 0; i < controls.length; i++) { + if (controls[i] instanceof PagedResultsResponseControl) { + PagedResultsResponseControl prrc = (PagedResultsResponseControl) controls[i]; + LOG.debug("END-OF-PAGE total: {}", prrc.getResultSize()); + return prrc.getCookie(); + } + } + } else { + LOG.debug("No controls were sent from the server"); + } + return null; + } + + private int processGroupMembers(Attribute groupMemberAttr, String groupFullName) throws Throwable { + NamingEnumeration userEnum = groupMemberAttr.getAll(); + int userCount = 0; + while (userEnum.hasMore()) { + String originalUserFullName = (String) userEnum.next(); + + if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) { + sourceGroupUsers.put(groupFullName, new HashSet<>()); + continue; + } + + userCount++; + + if (!userSearchEnabled) { + Map userAttrMap = new HashMap<>(); + String userName = getShortName(originalUserFullName); + + userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName); + userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName); + userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource); + userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl()); + + sourceUsers.put(originalUserFullName, userAttrMap); + + LOG.debug("As usersearch is disabled, adding user {} from group member attribute for group {}", userName, groupFullName); + } + + groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName); + } + return userCount; + } + + private void getRemainingGroupMembers(Map> largeGroups) throws Throwable { + NamingEnumeration groupSearchResultEnum = null; + + try { + createLdapContext(); + + int counter = 0; + + // Loop through each OU + for (String ou : largeGroups.keySet()) { + Set groupNames = largeGroups.get(ou); + + // Process each group individually + for (String groupName : groupNames) { + LOG.info("Processing large group: {}", groupName); + + // Build filter for THIS group only + String groupFilter = "(&(objectclass=" + groupObjectClass + ")"; + + // Add custom filter if present + if (groupSearchFilter != null && !groupSearchFilter.trim().isEmpty()) { + String customFilter = groupSearchFilter.trim(); + if (!customFilter.startsWith("(")) { + customFilter = "(" + customFilter + ")"; + } + groupFilter += customFilter; + } + + // Add the specific group name + groupFilter += "(CN=" + groupName + "))"; + + LOG.info("Group filter for {}: {}", groupName, groupFilter); + + try { + int rangeStep = 1500; + int start = 1500; + boolean lastRange = false; + + // Loop until we get all members for THIS group + while (!lastRange) { + // Build the range attribute name + String rangeAttr = "member;range=" + start + "-" + (start + rangeStep - 1); + String[] attributes = {rangeAttr}; + + LOG.info("Fetching range for {}: {}", groupName, rangeAttr); + + // Prepare search controls + SearchControls controls = new SearchControls(); + controls.setSearchScope(SearchControls.SUBTREE_SCOPE); + controls.setReturningAttributes(attributes); + + // Search for this specific group + groupSearchResultEnum = ldapContext.search(ou, groupFilter, controls); + + while (groupSearchResultEnum.hasMore()) { + final SearchResult result = groupSearchResultEnum.next(); + Attributes attrs = result.getAttributes(); + String groupFullName = result.getNameInNamespace(); + + LOG.info("Processing range for group: {}", groupFullName); + + NamingEnumeration attrEnum = attrs.getAll(); + + while (attrEnum.hasMore()) { + Attribute attr = attrEnum.next(); + String attrID = attr.getID(); + + LOG.info("Attribute ID: {}", attrID); + + // Check if this is the last range for THIS group + if (attrID.endsWith("*")) { + lastRange = true; + LOG.info("Last range reached for group: {}", groupName); + } + + int userCount = processGroupMembers(attr, groupFullName); + LOG.info("No. of members in the group {} for range {} = {}", groupFullName, rangeAttr, userCount); + } + + counter++; + } + + // Move to next range + start += rangeStep; + } + + LOG.info("Completed processing group: {} with total iterations: {}", groupName, counter); + } catch (Exception e) { + LOG.error("Error processing group {}: {}", groupName, e.getMessage(), e); + } + } + } + + LOG.info("Completed processing all large groups"); + } catch (Throwable t) { + LOG.error("LdapUserGroupBuilder.getRemainingGroupMembers() failed with exception:", t); + throw t; + } finally { + if (groupSearchResultEnum != null) { + groupSearchResultEnum.close(); + } + closeLdapContext(); + } + } + private String getFirstRDN(String name) { if (StringUtils.isEmpty(name)) { return null; @@ -1333,30 +1414,7 @@ private String getDNForMemberOf(String searchFilter) throws Throwable { } } } - - // Examine the paged results control response - Control[] controls = ldapContext.getResponseControls(); - - if (controls != null) { - for (Control control : controls) { - if (control instanceof PagedResultsResponseControl) { - PagedResultsResponseControl prrc = (PagedResultsResponseControl) control; - - total = prrc.getResultSize(); - - if (total != 0) { - LOG.debug("END-OF-PAGE total : {}", total); - } else { - LOG.debug("END-OF-PAGE total : unknown"); - } - - cookie = prrc.getCookie(); - } - } - } else { - LOG.debug("No controls were sent from the server"); - } - + cookie = handlePagedResults(); // Re-activate paged results if (pagedResultsEnabled) { LOG.debug("Fetched paged results round: {}", ++paged); @@ -1384,4 +1442,12 @@ private String getDNForMemberOf(String searchFilter) throws Throwable { return computedSearchFilter; } + + static { + try { + localHostname = java.net.InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + localHostname = "unknown"; + } + } } diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java index 08ca725000d..eb501ea12bc 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java @@ -137,6 +137,8 @@ public class UserGroupSyncConfig { private static final int DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS = 0; private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500; private static final boolean DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED = false; + private static final String LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED = "ranger.usersync.ldap.largegroupsync"; + private static final boolean DEFAULT_LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED = false; private static final boolean DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED = false; private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true; private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = true; @@ -1389,6 +1391,19 @@ public boolean isSyncSourceValidationEnabled() { return isSyncSourceValidationEnabled; } + public boolean isLargeGroupSyncEnabled() { + boolean largeGroupSyncEnabled; + String val = prop.getProperty(LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED); + + if (val == null || val.trim().isEmpty()) { + largeGroupSyncEnabled = DEFAULT_LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED; + } else { + largeGroupSyncEnabled = Boolean.valueOf(val); + } + + return largeGroupSyncEnabled; + } + private void init() { XMLUtils.loadConfig(DEFAULT_CONFIG_FILE, prop); XMLUtils.loadConfig(CORE_SITE_CONFIG_FILE, prop); diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java index ad8170efd11..b42700cd887 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java @@ -81,7 +81,6 @@ public class PolicyMgrUserGroupBuilder extends AbstractUserGroupSource implement public static final String PM_UPDATE_USERS_ROLES_URI = "/service/xusers/users/roleassignments"; /* ******************* */ - private static final String AUTHENTICATION_TYPE = "hadoop.security.authentication"; private static final String AUTH_KERBEROS = "kerberos"; private static final String KERBEROS_PRINCIPAL = "ranger.usersync.kerberos.principal"; private static final String KERBEROS_KEYTAB = "ranger.usersync.kerberos.keytab"; @@ -205,7 +204,7 @@ public synchronized void init() throws Throwable { String keyStoreType = config.getSSLKeyStoreType(); String trustStoreType = config.getSSLTrustStoreType(); - authenticationType = config.getProperty(AUTHENTICATION_TYPE, "simple"); + authenticationType = config.getProperty(UgsyncCommonConstants.AUTHENTICATION_TYPE, "simple"); try { principal = SecureClientLogin.getPrincipal(config.getProperty(KERBEROS_PRINCIPAL, ""), localHostname); diff --git a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java index 0b962439089..4c2af85af9d 100644 --- a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java +++ b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java @@ -31,15 +31,19 @@ import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo; import org.apache.ranger.unixusersync.config.UserGroupSyncConfig; import org.apache.ranger.usergroupsync.UserGroupSink; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import javax.naming.NamingEnumeration; +import javax.naming.NamingException; import javax.naming.directory.Attribute; import javax.naming.directory.Attributes; import javax.naming.directory.BasicAttribute; @@ -87,7 +91,8 @@ @CreateLdapConnectionPool(maxActive = 1, maxWait = 5000) @ApplyLdifFiles("ADSchema.ldif") public class TestLdapUserGroupBuilder extends AbstractLdapTestUnit { - private static final String STR_EPOCH = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(0)); + private static final String STR_EPOCH = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(0)); + private MockedStatic mockedConfig; @Test public void testA_init_and_isChanged() throws Exception { @@ -530,6 +535,7 @@ public void testW_getGroups_processes_groups_and_memberships() throws Throwable configureMinimalLdapConfig(); UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance(); cfg.setProperty("ranger.usersync.ldap.group.searchfilter", "cn=Group*"); + cfg.setProperty("ranger.usersync.ldap.largegroupsync", "true"); LdapUserGroupBuilder b = new LdapUserGroupBuilder(); b.init(); @@ -667,7 +673,7 @@ public void testX_goUpGroupHierarchyLdap_fetches_nested_groups_and_members() thr configureMinimalLdapConfig(); UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance(); cfg.setProperty("ranger.usersync.ldap.group.searchfilter", "cn=Nested*"); - + cfg.setProperty("ranger.usersync.ldap.groupattributelist", ""); LdapUserGroupBuilder b = new LdapUserGroupBuilder(); b.init(); @@ -677,15 +683,18 @@ public void testX_goUpGroupHierarchyLdap_fetches_nested_groups_and_members() thr fUserEnabled.setBoolean(b, false); // Prepare internal structures used by goUpGroupHierarchyLdap - Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable"); - Field srcUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceUsers"); - Field srcGroupsF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroups"); + Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable"); + Field srcUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceUsers"); + Field srcGroupsF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroups"); + Field srcGroupUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers"); gutF.setAccessible(true); srcUsersF.setAccessible(true); srcGroupsF.setAccessible(true); + srcGroupUsersF.setAccessible(true); gutF.set(b, HashBasedTable.create()); srcUsersF.set(b, new HashMap<>()); srcGroupsF.set(b, new HashMap<>()); + srcGroupUsersF.set(b, new HashMap<>()); // Parent group DN used to build filter final String parentGroupDn = "CN=Parent,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"; @@ -867,6 +876,259 @@ public SearchResult nextElement() { } } + @BeforeEach + void setupActiveState() { + // Mock the static method to always return true for active state + mockedConfig = Mockito.mockStatic(UserGroupSyncConfig.class, Mockito.CALLS_REAL_METHODS); + mockedConfig.when(UserGroupSyncConfig::isUgsyncServiceActive).thenReturn(true); + } + + @AfterEach + void tearDownActiveState() { + if (mockedConfig != null) { + mockedConfig.close(); + } + } + + @Test + void testZ_largeGroupSyncEnabled_detectsInitialRange() throws Throwable { + resetConfig(); + configureMinimalLdapConfig(); + UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance(); + // FIX: Use the correct property name + cfg.setProperty("ranger.usersync.ldap.largegroupsync", "true"); + + LdapUserGroupBuilder b = new LdapUserGroupBuilder(); + b.init(); + + // Verify that large group sync is enabled in config + assertTrue(cfg.isLargeGroupSyncEnabled(), "Large group sync should be enabled"); + + // Verify the processGroupMembers method exists + Method processGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers", Attribute.class, String.class); + assertNotNull(processGroupMembers, "processGroupMembers method should exist"); + + // Verify getRemainingGroupMembers method exists (new method for large groups) + Method getRemainingGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("getRemainingGroupMembers", Map.class); + assertNotNull(getRemainingGroupMembers, "getRemainingGroupMembers method should exist"); + + // Verify that group search controls include the ranged attribute when large group sync is enabled + Field groupSearchControlsF = LdapUserGroupBuilder.class.getDeclaredField("groupSearchControls"); + groupSearchControlsF.setAccessible(true); + SearchControls controls = (SearchControls) groupSearchControlsF.get(b); + String[] returningAttrs = controls.getReturningAttributes(); + + // Check if "member;range=0-1499" is in the returning attributes + boolean hasRangeAttr = false; + for (String attr : returningAttrs) { + if (attr.equals("member;range=0-1499")) { + hasRangeAttr = true; + break; + } + } + assertFalse(hasRangeAttr, "Should NOT include member;range=0-1499 in initial search - it prevents AD from returning regular member attribute for small groups"); + } + + @Test + void testZA_getRemainingGroupMembers_handles_ranged_attributes() throws Throwable { + resetConfig(); + configureMinimalLdapConfig(); + UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance(); + cfg.setProperty("ranger.usersync.ldap.largegroupsync", "true"); + + LdapUserGroupBuilder b = new LdapUserGroupBuilder(); + b.init(); + + // Prepare internal structures + Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable"); + Field srcUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceUsers"); + Field srcGroupsF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroups"); + gutF.setAccessible(true); + srcUsersF.setAccessible(true); + srcGroupsF.setAccessible(true); + gutF.set(b, HashBasedTable.create()); + srcUsersF.set(b, new HashMap<>()); + srcGroupsF.set(b, new HashMap<>()); + + Method getRemainingMembers = LdapUserGroupBuilder.class.getDeclaredMethod("getRemainingGroupMembers", Map.class); + getRemainingMembers.setAccessible(true); + + // This will throw because it tries to create LDAP context + // Test that it throws Throwable (which is expected without a real LDAP server) + Map> emptyMap = new HashMap<>(); + assertThrows(Throwable.class, () -> getRemainingMembers.invoke(b, emptyMap)); + } + + @Test + void testZB_processGroupMembers_handles_empty_members() throws Throwable { + resetConfig(); + configureMinimalLdapConfig(); + + LdapUserGroupBuilder b = new LdapUserGroupBuilder(); + b.init(); + + // Prepare internal structures + Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable"); + Field srcGroupUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers"); + gutF.setAccessible(true); + srcGroupUsersF.setAccessible(true); + gutF.set(b, HashBasedTable.create()); + srcGroupUsersF.set(b, new HashMap<>()); + + Method processGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers", Attribute.class, String.class); + processGroupMembers.setAccessible(true); + + final String groupDn = "CN=TestGroup,OU=Groups,DC=example,DC=com"; + + // Create attribute with mix of empty and valid members + BasicAttribute members = new BasicAttribute("member"); + members.add(""); // empty member - causes sourceGroupUsers.put and continue + members.add("uid=validuser,ou=people,dc=example,dc=com"); + + int count = (Integer) processGroupMembers.invoke(b, members, groupDn); + + // Only the valid member is counted (empty member triggers continue before userCount++) + assertEquals(1, count); + + @SuppressWarnings("unchecked") + Table gut = (Table) gutF.get(b); + + // Only valid user should be in the table + assertFalse(gut.contains(groupDn, "")); + assertTrue(gut.contains(groupDn, "uid=validuser,ou=people,dc=example,dc=com")); + + // sourceGroupUsers should have been set (from the empty member check) + @SuppressWarnings("unchecked") + Map> srcGroupUsers = (Map>) srcGroupUsersF.get(b); + assertTrue(srcGroupUsers.containsKey(groupDn)); + } + + @Test + void testZC_largeGroupSync_disabled_skips_remaining_members() throws Throwable { + resetConfig(); + configureMinimalLdapConfig(); + UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance(); + cfg.setProperty("ranger.usersync.ldap.group.searchfilter", "cn=LargeGroup*"); + cfg.setProperty("ranger.usersync.large.group.sync.enabled", "false"); + + LdapUserGroupBuilder b = new LdapUserGroupBuilder(); + b.init(); + + // Verify that large group sync is disabled + assertFalse(cfg.isLargeGroupSyncEnabled()); + + // Test that with disabled large group sync, processGroupMembers still works normally + Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable"); + Field srcGroupUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers"); + gutF.setAccessible(true); + srcGroupUsersF.setAccessible(true); + gutF.set(b, HashBasedTable.create()); + srcGroupUsersF.set(b, new HashMap<>()); + + Method processGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers", Attribute.class, String.class); + processGroupMembers.setAccessible(true); + + final String groupDn = "CN=NormalGroup,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com"; + BasicAttribute members = new BasicAttribute("member"); + members.add("uid=user1,ou=people,dc=example,dc=com"); + members.add("uid=user2,ou=people,dc=example,dc=com"); + + int count = (Integer) processGroupMembers.invoke(b, members, groupDn); + assertEquals(2, count); + } + + @Test + void testZD_getRemainingGroupMembers_handles_exceptions() throws Throwable { + resetConfig(); + configureMinimalLdapConfig(); + + LdapUserGroupBuilder b = new LdapUserGroupBuilder(); + b.init(); + + // Prepare internal structures + Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable"); + gutF.setAccessible(true); + gutF.set(b, HashBasedTable.create()); + + try (MockedConstruction mocked = Mockito.mockConstruction(InitialLdapContext.class, (mock, ctx) -> { + Mockito.when(mock.search(Mockito.anyString(), Mockito.anyString(), Mockito.any(SearchControls.class))) + .thenThrow(new NamingException("Simulated LDAP error")); + Mockito.when(mock.getResponseControls()).thenReturn(null); + })) { + Map> largeGroups = new HashMap<>(); + Set groupNames = new HashSet<>(); + groupNames.add("FailingGroup"); + largeGroups.put("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com", groupNames); + + Method getRemainingMembers = LdapUserGroupBuilder.class.getDeclaredMethod("getRemainingGroupMembers", Map.class); + getRemainingMembers.setAccessible(true); + + // Should not throw, but log error and continue + assertDoesNotThrow(() -> getRemainingMembers.invoke(b, largeGroups)); + } + } + + @Test + void testZE_processGroupMembers_handles_members_correctly() throws Throwable { + resetConfig(); + configureMinimalLdapConfig(); + + LdapUserGroupBuilder b = new LdapUserGroupBuilder(); + b.init(); + + // Disable user search so members are added to sourceUsers directly + Field fUserEnabled = LdapUserGroupBuilder.class.getDeclaredField("userSearchEnabled"); + Field currentSyncSourceF = LdapUserGroupBuilder.class.getDeclaredField("currentSyncSource"); + fUserEnabled.setAccessible(true); + currentSyncSourceF.setAccessible(true); + fUserEnabled.setBoolean(b, false); + + // Ensure currentSyncSource is set + if (currentSyncSourceF.get(b) == null) { + currentSyncSourceF.set(b, "LDAP"); + } + + // Prepare internal structures + Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable"); + Field srcUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceUsers"); + Field srcGroupUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers"); + gutF.setAccessible(true); + srcUsersF.setAccessible(true); + srcGroupUsersF.setAccessible(true); + gutF.set(b, HashBasedTable.create()); + srcUsersF.set(b, new HashMap<>()); + srcGroupUsersF.set(b, new HashMap<>()); + + Method processGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers", Attribute.class, String.class); + processGroupMembers.setAccessible(true); + + final String groupDn = "CN=TestGroup,OU=Groups,DC=example,DC=com"; + + // Test with a mix of valid and edge case members + BasicAttribute members = new BasicAttribute("member"); + members.add("uid=user1,ou=people,dc=example,dc=com"); + members.add("uid=user2,ou=people,dc=example,dc=com"); + members.add("uid=user3,ou=people,dc=example,dc=com"); + + int count = (Integer) processGroupMembers.invoke(b, members, groupDn); + assertEquals(3, count); + + @SuppressWarnings("unchecked") + Table gut = (Table) gutF.get(b); + + // All valid users should be in the table + assertTrue(gut.contains(groupDn, "uid=user1,ou=people,dc=example,dc=com")); + assertTrue(gut.contains(groupDn, "uid=user2,ou=people,dc=example,dc=com")); + assertTrue(gut.contains(groupDn, "uid=user3,ou=people,dc=example,dc=com")); + + // When userSearchEnabled is false, users should also be added to sourceUsers + @SuppressWarnings("unchecked") + Map> srcUsers = (Map>) srcUsersF.get(b); + assertTrue(srcUsers.containsKey("uid=user1,ou=people,dc=example,dc=com")); + assertTrue(srcUsers.containsKey("uid=user2,ou=people,dc=example,dc=com")); + assertTrue(srcUsers.containsKey("uid=user3,ou=people,dc=example,dc=com")); + } + private static void configureMinimalLdapConfig() throws Exception { UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance(); cfg.setProperty("ranger.usersync.ldap.url", "ldap://127.0.0.1:389");