1 package edu.internet2.middleware.grouper.pspng;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 import java.io.IOException;
20 import java.io.Reader;
21 import java.io.StringReader;
22 import java.util.*;
23
24 import com.unboundid.ldap.sdk.Filter;
25 import com.unboundid.ldap.sdk.LDAPException;
26 import org.apache.commons.lang.StringUtils;
27 import org.ldaptive.*;
28 import org.ldaptive.io.LdifReader;
29
30 import edu.internet2.middleware.subject.Subject;
31 import static edu.internet2.middleware.grouper.pspng.PspUtils.*;
32
33
34
35
36
37
38
39
40
41
42 public class LdapGroupProvisioner extends LdapProvisioner<LdapGroupProvisionerConfiguration> {
43
44 public LdapGroupProvisioner(String provisionerName, LdapGroupProvisionerConfiguration config, boolean fullSyncMode) {
45 super(provisionerName, config, fullSyncMode);
46
47 LOG.debug("Constructing LdapGroupProvisioner: {}", provisionerName);
48 }
49
50 public static Class<? extends ProvisionerConfiguration> getPropertyClass() {
51 return LdapGroupProvisionerConfiguration.class;
52 }
53
54
55 @Override
56 protected void addMembership(GrouperGroupInfo grouperGroupInfo, LdapGroup ldapGroup,
57 Subject subject, LdapUser ldapUser) throws PspException {
58
59
60
61
62
63 if ( ldapUser == null ) {
64 LOG.warn("{}: Skipping adding membership to group {} because ldap user does not exist: {}",
65 new Object[]{getDisplayName(), grouperGroupInfo, subject});
66 return;
67 }
68
69 if ( ldapGroup == null ) {
70
71
72
73
74
75 ldapGroup = createGroup(grouperGroupInfo, Arrays.asList(subject));
76 cacheGroup(grouperGroupInfo, ldapGroup);
77 }
78 else {
79 String membershipAttributeValue = evaluateJexlExpression("MemberAttributeValue", config.getMemberAttributeValueFormat(), subject, ldapUser, grouperGroupInfo, ldapGroup);
80 if ( membershipAttributeValue != null ) {
81 scheduleGroupModification(grouperGroupInfo, ldapGroup, AttributeModificationType.ADD, Arrays.asList(membershipAttributeValue));
82 }
83 }
84 }
85
86
87 protected void scheduleGroupModification(GrouperGroupInfo grouperGroupInfo, LdapGroup ldapGroup, AttributeModificationType modType, Collection<String> membershipValuesToChange) {
88 String attributeName = config.getMemberAttributeName();
89
90 for ( String value : membershipValuesToChange )
91
92 LOG.info("Will change LDAP: {} {} {} {} of {}",
93 new Object[] {modType, value,
94 modType == AttributeModificationType.ADD ? "to" : "from",
95 attributeName, ldapGroup});
96
97 scheduleLdapModification(
98 new ModifyRequest(
99 ldapGroup.getLdapObject().getDn(),
100 new AttributeModification(
101 modType,
102 new LdapAttribute(attributeName, membershipValuesToChange.toArray(new String[0])))));
103 }
104
105 @Override
106 protected void deleteMembership(GrouperGroupInfo grouperGroupInfo, LdapGroup ldapGroup ,
107 Subject subject, LdapUser ldapUser) throws PspException {
108 if ( ldapGroup == null ) {
109 LOG.warn("{}: Ignoring request to remove {} from a group that doesn't exist: {}",
110 new Object[]{getDisplayName(), subject.getId(), grouperGroupInfo});
111 return;
112 }
113
114 if ( ldapUser == null ) {
115 LOG.warn("{}: Skipping removing membership from group {} because ldap user does not exist: {}",
116 new Object[]{getDisplayName(), grouperGroupInfo, subject});
117 return;
118 }
119
120
121
122
123
124 String membershipAttributeValue = evaluateJexlExpression("MemberAttributeValue", config.getMemberAttributeValueFormat(), subject, ldapUser, grouperGroupInfo, ldapGroup);
125
126 if ( membershipAttributeValue != null ) {
127 scheduleGroupModification(grouperGroupInfo, ldapGroup, AttributeModificationType.REMOVE, Arrays.asList(membershipAttributeValue));
128 }
129 }
130
131 @Override
132 protected boolean doFullSync(
133 GrouperGroupInfo grouperGroupInfo, LdapGroup ldapGroup ,
134 Set<Subject> correctSubjects, Map<Subject, LdapUser> tsUserMap,
135 Set<LdapUser> correctTSUsers,
136 JobStatistics stats) throws PspException {
137
138 stats.totalCount.set(correctSubjects.size());
139
140
141
142 if ( ldapGroup != null )
143 ldapGroup.getLdapObject().getStringValues(config.getMemberAttributeName());
144
145
146 if ( ldapGroup == null ) {
147
148
149 if ( config.areEmptyGroupsSupported() ) {
150 if ( correctSubjects.size() == 0 ) {
151 LOG.info("{}: Nothing to do because empty group already not present in ldap system", getDisplayName() );
152 return false;
153 }
154 }
155
156 ldapGroup = createGroup(grouperGroupInfo, correctSubjects);
157 stats.insertCount.set(correctSubjects.size());
158
159
160 if ( ldapGroup != null ) {
161 cacheGroup(grouperGroupInfo, ldapGroup);
162 }
163 return true;
164 } else {
165
166 ldapGroup = updateGroupFromTemplate(grouperGroupInfo, ldapGroup);
167 cacheGroup(grouperGroupInfo, ldapGroup);
168 }
169
170
171 if ( !config.areEmptyGroupsSupported() && correctSubjects.size() == 0 ) {
172 LOG.info("{}: Deleting empty group because schema requires its member attribute", getDisplayName());
173 deleteGroup(grouperGroupInfo, ldapGroup);
174
175
176 Collection<String> membershipValues = ldapGroup.getLdapObject().getStringValues(config.getMemberAttributeName());
177 stats.deleteCount.set(membershipValues.size());
178
179 return true;
180 }
181
182 Set<String> correctMembershipValues = getStringSet(config.isMemberAttributeCaseSensitive());
183
184 for ( Subject correctSubject: correctSubjects ) {
185 String membershipAttributeValue = evaluateJexlExpression("MemberAttributeValue", config.getMemberAttributeValueFormat(), correctSubject, tsUserMap.get(correctSubject), grouperGroupInfo, ldapGroup);
186
187 if ( membershipAttributeValue != null ) {
188 correctMembershipValues.add(membershipAttributeValue);
189 }
190 }
191
192 Collection<String> currentMembershipValues = getStringSet(config.isMemberAttributeCaseSensitive(), ldapGroup.getLdapObject().getStringValues(config.getMemberAttributeName()));
193
194 LOG.info("{}: Full-sync comparison for {}: Target-subject count: Correct/Actual: {}/{}",
195 new Object[] {getDisplayName(), grouperGroupInfo, correctMembershipValues.size(), currentMembershipValues.size()});
196
197 LOG.debug("{}: Full-sync comparison: Correct: {}", getDisplayName(), correctMembershipValues);
198 LOG.debug("{}: Full-sync comparison: Actual: {}", getDisplayName(), currentMembershipValues);
199
200
201 Collection<String> extraValues = subtractStringCollections(
202 config.isMemberAttributeCaseSensitive(), currentMembershipValues, correctMembershipValues);
203
204 stats.deleteCount.set(extraValues.size());
205
206 LOG.info("{}: Group {} has {} extra values",
207 new Object[] {getDisplayName(), grouperGroupInfo, extraValues.size()});
208 if ( extraValues.size() > 0 ) {
209 getLdapSystem().performLdapModify(
210 new ModifyRequest(
211 ldapGroup.dn,
212 new AttributeModification(
213 AttributeModificationType.REMOVE,
214 new LdapAttribute(config.getMemberAttributeName(),extraValues.toArray(new String[0])))),
215 config.isMemberAttributeCaseSensitive(),
216 true);
217 }
218
219
220 Collection<String> missingValues = subtractStringCollections(
221 config.isMemberAttributeCaseSensitive(), correctMembershipValues, currentMembershipValues);
222
223 stats.insertCount.set(missingValues.size());
224
225 LOG.info("{}: Group {} has {} missing values",
226 new Object[]{getDisplayName(), grouperGroupInfo, missingValues.size()});
227 if ( missingValues.size() > 0 ) {
228 getLdapSystem().performLdapModify(
229 new ModifyRequest(
230 ldapGroup.dn,
231 new AttributeModification(
232 AttributeModificationType.ADD,
233 new LdapAttribute(config.getMemberAttributeName(),missingValues.toArray(new String[0])))),
234 config.isMemberAttributeCaseSensitive(),
235 true);
236
237 }
238
239 return extraValues.size()>0 || missingValues.size()>0;
240 }
241
242
243
244
245
246
247
248
249 protected LdapGroupg/LdapGroup.html#LdapGroup">LdapGroup updateGroupFromTemplate(GrouperGroupInfo grouperGroupInfo, LdapGroup existingLdapGroup) throws PspException {
250 LOG.debug("{}: Making sure (non-membership) attributes of group are up to date: {}", getDisplayName(), existingLdapGroup.dn);
251
252 try {
253 String ldifFromTemplate = getGroupLdifFromTemplate(grouperGroupInfo);
254 LdapEntry ldapEntryFromTemplate = getLdapEntryFromLdif(ldifFromTemplate);
255
256 ensureLdapOusExist(ldapEntryFromTemplate.getDn(), false);
257 if ( getLdapSystem().makeLdapObjectCorrect(ldapEntryFromTemplate, existingLdapGroup.ldapObject.ldapEntry, config.isMemberAttributeCaseSensitive()) ) {
258 LdapGroup result = fetchTargetSystemGroup(grouperGroupInfo);
259 return result;
260 }
261 else {
262 return existingLdapGroup;
263 }
264 }
265 catch (PspException e) {
266 LOG.error("{}: Problem checking and updating group's template attributes", getDisplayName(), e);
267 throw e;
268 }
269 catch (IOException e) {
270 LOG.error("{}: Problem checking and updating group's tempalte attributes", getDisplayName(), e);
271 throw new PspException("IO Exception while checking and updating group's template attributes", e);
272 }
273 }
274
275
276
277 @Override
278 protected void doFullSync_cleanupExtraGroups(JobStatistics stats) throws PspException {
279
280
281 String filterString = config.getAllGroupSearchFilter();
282 if ( StringUtils.isEmpty(filterString) ) {
283 LOG.error("{}: Cannot cleanup extra groups without a configured all-group search filter", getDisplayName());
284 return;
285 }
286
287 String baseDn = config.getGroupSearchBaseDn();
288
289 if ( StringUtils.isEmpty(baseDn)) {
290 LOG.error("{}: Cannot cleanup extra groups without a configured group-search base dn", getDisplayName());
291 return;
292 }
293
294
295 List<LdapObject> allProvisionedGroups
296 = getLdapSystem().performLdapSearchRequest(
297 -1, baseDn, SearchScope.SUBTREE,
298 Arrays.asList(getLdapAttributesToFetch()), filterString);
299
300
301
302
303 Collection<GrouperGroupInfo> groupsThatShouldBeProvisioned = getAllGroupsForProvisioner();
304 Map<GrouperGroupInfo, LdapGroup> ldapGroupsThatShouldBeProvisioned = fetchTargetSystemGroupsInBatches(groupsThatShouldBeProvisioned);
305
306 Set<String> correctGroupDNs = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
307 for(LdapGroup correctLdapGroup : ldapGroupsThatShouldBeProvisioned.values()) {
308 String correctLdapGroupDn = correctLdapGroup.getLdapObject().getDn();
309 correctGroupDNs.add(correctLdapGroupDn);
310 }
311
312
313 List<LdapObject> groupsToDelete = new ArrayList<LdapObject>();
314 for (LdapObject aProvisionedGroup : allProvisionedGroups) {
315 if ( ! correctGroupDNs.contains(aProvisionedGroup.getDn()) ) {
316 groupsToDelete.add(aProvisionedGroup);
317 }
318 }
319
320 LOG.info("{}: There are {} groups that we should delete", getDisplayName(), groupsToDelete.size());
321
322 for ( LdapObject groupToRemove : groupsToDelete ) {
323 int numMembershipsBeingDeleted = groupToRemove.getStringValues(config.getMemberAttributeName()).size();
324 stats.deleteCount.addAndGet(numMembershipsBeingDeleted);
325
326 getLdapSystem().performLdapDelete(groupToRemove.getDn());
327 }
328
329 }
330
331
332 @Override
333 protected LdapGroup createGroup(GrouperGroupInfo grouperGroup, Collection<Subject> initialMembers) throws PspException {
334 if ( !config.areEmptyGroupsSupported() && initialMembers.size() == 0 ) {
335 LOG.warn("Not Creating LDAP group because empty groups are not supported: {}", grouperGroup);
336 return null;
337 }
338
339 LOG.info("Creating LDAP group for GrouperGroup: {} ", grouperGroup);
340 String ldif = getGroupLdifFromTemplate(grouperGroup);
341
342
343 if ( initialMembers != null && initialMembers.size() > 0 ) {
344
345
346 Collection<String> membershipValues = new HashSet<String>(initialMembers.size());
347 for ( Subject subject : initialMembers ) {
348 LdapUser ldapUser = getTargetSystemUser(subject);
349 if ( ldapUser != null ) {
350 String membershipAttributeValue = evaluateJexlExpression("MemberAttributeValue", config.getMemberAttributeValueFormat(), subject, ldapUser, grouperGroup, null);
351 if ( membershipAttributeValue != null ) {
352 membershipValues.add(membershipAttributeValue);
353 }
354 }
355 }
356
357 StringBuilder ldifForMemberships = new StringBuilder();
358 for ( String attributeValue : membershipValues ) {
359 ldifForMemberships.append(String.format("%s: %s\n", config.getMemberAttributeName(), attributeValue));
360 }
361 ldif = ldif.concat("\n");
362 ldif = ldif.concat(ldifForMemberships.toString());
363 }
364
365 Connection conn = getLdapSystem().getLdapConnection();
366 try {
367 LOG.debug("{}: LDIF for new group (with partial DN): {}", getDisplayName(), ldif.replaceAll("\\n", "||"));
368 LdapEntry ldifEntry = getLdapEntryFromLdif(ldif);
369
370
371 for ( String attributeName : ldifEntry.getAttributeNames() ) {
372 LdapAttribute attribute = ldifEntry.getAttribute(attributeName);
373 if ( LdapSystem.attributeHasNoValues(attribute) ) {
374 LOG.warn("{}: LDIF for new group did not define any values for {}", getDisplayName(), attributeName);
375 ldifEntry.removeAttribute(attributeName);
376 }
377 }
378 LOG.debug("{}: Adding group: {}", getDisplayName(), ldifEntry);
379
380 performLdapAdd(ldifEntry);
381
382
383 LOG.debug("Reading group that was just added to ldap server: {}", grouperGroup);
384 LdapGroup result = fetchTargetSystemGroup(grouperGroup);
385
386 if ( result == null ) {
387 LOG.error("{}: Group could not be found after it was created: {}", getDisplayName(), grouperGroup);
388 }
389 return result;
390 } catch (PspException e) {
391 LOG.error("Problem while creating new group: {}", ldif, e);
392 throw e;
393 } catch ( IOException e ) {
394 LOG.error("IO problem while creating group: {}", ldif, e);
395 throw new PspException("IO problem while creating group: %s", e.getMessage());
396 }
397 finally {
398 conn.close();
399 }
400 }
401
402
403
404
405
406
407
408
409
410 private LdapEntry getLdapEntryFromLdif(String ldif) throws IOException {
411 Reader reader = new StringReader(ldif);
412 LdifReader ldifReader = new LdifReader(reader);
413 SearchResult ldifResult = ldifReader.read();
414 LdapEntry ldifEntry = ldifResult.getEntry();
415
416
417 String actualDn = String.format("%s,%s", ldifEntry.getDn(),config.getGroupCreationBaseDn());
418 ldifEntry.setDn(actualDn);
419 return ldifEntry;
420 }
421
422
423
424
425
426
427
428 private String getGroupLdifFromTemplate(GrouperGroupInfo grouperGroup) throws PspException {
429 String ldif = config.getGroupCreationLdifTemplate();
430 ldif = ldif.replaceAll("\\|\\|", "\n");
431 ldif = evaluateJexlExpression("GroupTemplate", ldif, null, null, grouperGroup, null);
432 ldif = sanityCheckDnAttributesOfLdif(ldif, "Group ldif for %s", grouperGroup);
433
434 return ldif;
435 }
436
437 @Override
438 protected Map<GrouperGroupInfo, LdapGroup> fetchTargetSystemGroups(
439 Collection<GrouperGroupInfo> grouperGroupsToFetch) throws PspException {
440 if ( grouperGroupsToFetch.size() > config.getGroupSearch_batchSize() )
441 throw new IllegalArgumentException("LdapGroupProvisioner.fetchTargetSystemGroups: invoked with too many groups to fetch");
442
443
444
445 String[] returnAttributes = getLdapAttributesToFetch();
446
447 if ( grouperGroupsToFetch.size() > 1 && config.isBulkGroupSearchingEnabled() ) {
448 StringBuilder combinedLdapFilter = new StringBuilder();
449
450
451 combinedLdapFilter.append("(|");
452
453 for (GrouperGroupInfo grouperGroup : grouperGroupsToFetch) {
454 SearchFilter f = getGroupLdapFilter(grouperGroup);
455 String groupFilterString = f.format();
456
457
458 if (groupFilterString.startsWith("("))
459 combinedLdapFilter.append(groupFilterString);
460 else
461 combinedLdapFilter.append('(').append(groupFilterString).append(')');
462 }
463 combinedLdapFilter.append(')');
464
465
466 List<LdapObject> searchResult;
467
468 LOG.debug("{}: Searching for {} groups with:: {}",
469 new Object[]{getDisplayName(), grouperGroupsToFetch.size(), combinedLdapFilter});
470
471 try {
472 searchResult = getLdapSystem().performLdapSearchRequest(
473 -1, config.getGroupSearchBaseDn(), SearchScope.SUBTREE,
474 Arrays.asList(returnAttributes),
475 combinedLdapFilter.toString());
476 } catch (PspException e) {
477 LOG.error("Problem fetching groups with filter '{}' on base '{}'",
478 new Object[]{combinedLdapFilter, config.getGroupSearchBaseDn(), e});
479 throw e;
480 }
481
482 LOG.debug("{}: Group search returned {} groups", getDisplayName(), searchResult.size());
483
484
485
486
487 Map<GrouperGroupInfo, LdapGroup> result = new HashMap<GrouperGroupInfo, LdapGroup>();
488
489 Set<LdapObject> matchedFetchResults = new HashSet<LdapObject>();
490
491
492 for (GrouperGroupInfo groupToFetch : grouperGroupsToFetch) {
493 SearchFilter f = getGroupLdapFilter(groupToFetch);
494
495 for (LdapObject aFetchedLdapObject : searchResult) {
496 if (aFetchedLdapObject.matchesLdapFilter(f)) {
497 result.put(groupToFetch, new LdapGroup(aFetchedLdapObject));
498 matchedFetchResults.add(aFetchedLdapObject);
499 break;
500 }
501 }
502 }
503
504 Set<LdapObject> unmatchedFetchResults = new HashSet<LdapObject>(searchResult);
505 unmatchedFetchResults.removeAll(matchedFetchResults);
506
507
508 if ( unmatchedFetchResults.size() == 0 ) {
509 return result;
510 }
511 else {
512 for (LdapObject unmatchedFetchResult : unmatchedFetchResults) {
513 LOG.warn("{}: Bulk fetch failed (returned unmatchable group data). "
514 + "This can be caused by searching for a DN with escaping or by singleGroupSearchFilter ({}) that are not included "
515 + "in groupSearchAttributes ({})?): {}",
516 new Object[]{getDisplayName(), config.getSingleGroupSearchFilter(), config.getGroupSearchAttributes(), unmatchedFetchResult.getDn()});
517 }
518 LOG.warn("{}: Slower fetching will be attempted", getDisplayName());
519
520
521
522 }
523 }
524
525
526 Map<GrouperGroupInfo, LdapGroup> result = new HashMap<GrouperGroupInfo, LdapGroup>();
527
528 for (GrouperGroupInfo grouperGroup : grouperGroupsToFetch) {
529 SearchFilter groupLdapFilter = getGroupLdapFilter(grouperGroup);
530 try {
531 LOG.debug("{}: Searching for group {} with:: {}",
532 new Object[]{getDisplayName(), grouperGroup, groupLdapFilter});
533
534
535 List<LdapObject> searchResult = getLdapSystem().performLdapSearchRequest(
536 -1, config.getGroupSearchBaseDn(), SearchScope.SUBTREE,
537 Arrays.asList(returnAttributes),
538 groupLdapFilter);
539
540 if (searchResult.size() == 1) {
541 LdapObject ldapObject = searchResult.iterator().next();
542 LOG.debug("{}: Group search returned {}", getDisplayName(), ldapObject.getDn());
543 result.put(grouperGroup, new LdapGroup(ldapObject));
544 }
545 else if ( searchResult.size() > 1 ){
546 LOG.error("{}: Search for group {} with '{}' returned multiple matches: {}",
547 new Object[]{getDisplayName(), grouperGroup, groupLdapFilter, searchResult});
548 throw new PspException("Search for ldap group returned multiple matches");
549 }
550 else if ( searchResult.size() == 0 ) {
551
552 LOG.debug("{}: Group search did not return any results", getDisplayName());
553 }
554 } catch (PspException e) {
555 LOG.error("{}: Problem fetching group with filter '{}' on base '{}'",
556 new Object[]{getDisplayName(), groupLdapFilter, config.getGroupSearchBaseDn(), e});
557 throw e;
558 }
559 }
560
561 return result;
562 }
563
564 private String[] getLdapAttributesToFetch() {
565 String returnAttributes[] = config.getGroupSearchAttributes();
566 if ( fullSyncMode ) {
567 LOG.debug("Fetching membership attribute, too");
568
569 returnAttributes = Arrays.copyOf(returnAttributes, returnAttributes.length + 1);
570 returnAttributes[returnAttributes.length-1] = config.getMemberAttributeName();
571 } else {
572 LOG.debug("Fetching without membership attribute");
573 }
574 return returnAttributes;
575 }
576
577
578 private SearchFilter getGroupLdapFilter(GrouperGroupInfo grouperGroup) throws PspException {
579 String result = evaluateJexlExpression("SingleGroupSearchFilter", config.getSingleGroupSearchFilter(), null, null, grouperGroup, null);
580 if ( StringUtils.isEmpty(result) )
581 throw new RuntimeException("Group searching requires singleGroupSearchFilter to be configured correctly");
582
583
584 String filterPieces[] = result.split("\\|\\|");
585 SearchFilter filter = new SearchFilter(filterPieces[0]);
586
587 if ( filterPieces.length == 1 ) {
588 try {
589
590 Filter.create(result);
591 }
592 catch (LDAPException e) {
593 LOG.warn("{}: Group ldap filter was invalid. " +
594 "Perhaps its filter clauses needed to be escaped with utils.escapeLdapFilter or use ldap-filter positional parameters. " +
595 "Group={}. Bad filter={}. ",
596 new Object[]{getDisplayName(), grouperGroup, result});
597
598
599
600 }
601 } else {
602
603
604 for (int i = 1; i < filterPieces.length; i++)
605 filter.setParameter(i - 1, filterPieces[i].trim());
606 }
607
608 LOG.trace("{}: Filter for group {}: {}",
609 new Object[] {getDisplayName(), grouperGroup, filter});
610
611 return filter;
612 }
613
614
615 @Override
616 protected void deleteGroup(GrouperGroupInfo grouperGroupInfo, LdapGroup ldapGroup)
617 throws PspException {
618 if ( ldapGroup == null ) {
619 LOG.warn("Nothing to do: Unable to delete group {} because the group wasn't found on target system", grouperGroupInfo);
620 return;
621 }
622
623 String dn = ldapGroup.getLdapObject().getDn();
624
625 LOG.info("Deleting group {} by deleting DN {}", grouperGroupInfo, dn);
626
627 getLdapSystem().performLdapDelete(dn);;
628 }
629 }