Index: lams_central/src/java/org/lamsfoundation/lams/webservice/SPEnrolmentServlet.java =================================================================== diff -u -r22cb2a4fe2cc019a06038c30e463707be0410946 -r0580bae0e4b3b3e5671bcf05f683f0f34b88b832 --- lams_central/src/java/org/lamsfoundation/lams/webservice/SPEnrolmentServlet.java (.../SPEnrolmentServlet.java) (revision 22cb2a4fe2cc019a06038c30e463707be0410946) +++ lams_central/src/java/org/lamsfoundation/lams/webservice/SPEnrolmentServlet.java (.../SPEnrolmentServlet.java) (revision 0580bae0e4b3b3e5671bcf05f683f0f34b88b832) @@ -26,18 +26,24 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.Spliterator; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -70,6 +76,7 @@ import org.lamsfoundation.lams.util.Configuration; import org.lamsfoundation.lams.util.ConfigurationKeys; import org.lamsfoundation.lams.util.HashUtil; +import org.lamsfoundation.lams.util.WebUtil; import org.lamsfoundation.lams.util.hibernate.HibernateSessionManager; import org.springframework.web.context.support.WebApplicationContextUtils; @@ -105,6 +112,9 @@ private static final String FILE_INPUT_DEFAULT_NAME = "LAMS-OUTPUT.csv"; private static final String FILE_INPUT_PARAM = "file-input"; + private static final int THREADS_DEFAULT_VALUE = 4; + private static final String THREADS_PARAM = "threads"; + private static final String DELIMITER = "\\|"; private static final String INTEGRATED_SERVER_NAME = "saml"; @@ -113,14 +123,26 @@ private static ILogEventService logEventService = null; private static IIntegrationService integrationService = null; + private int threadCount = THREADS_DEFAULT_VALUE; + private ExecutorService executor = null; + @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + Integer threadCount = WebUtil.readIntParam(request, THREADS_PARAM, true); + if (threadCount != null && threadCount > 0) { + this.threadCount = threadCount; + } + executor = Executors.newFixedThreadPool(this.threadCount); + // check if path to input file is provided as GET parameter // if not, use default one of /LAMS-OUTPUT.csv String fileInputParam = request.getParameter(FILE_INPUT_PARAM); Path fileInput = StringUtils.isBlank(fileInputParam) ? Paths.get(Configuration.get(ConfigurationKeys.LAMS_TEMP_DIR), FILE_INPUT_DEFAULT_NAME) : Paths.get(fileInputParam); + if (!Files.isReadable(fileInput)) { + throw new IOException("File not readable: " + fileInput.toAbsolutePath().toString()); + } // run processing in a separate thread as it can take a while and request would time out new Thread(() -> { @@ -154,11 +176,11 @@ }).collect(Collectors.toList()); if (allLines.isEmpty()) { - throw new ServletException("File is empty"); + throw new IOException("File is empty"); } // map of user login -> user ID - Map userIDs = new HashMap<>(); + Map userIDs = new ConcurrentHashMap<>(); // find sysadmin as he/she will be the creator of organisations Organisation rootOrganisation = userManagementService.getRootOrganisation(); @@ -169,10 +191,9 @@ ConcurrentMap allParsedUserMapping = allLines.parallelStream().unordered() .collect(Collectors.toConcurrentMap( elem -> elem.get(6).equals(Mode.STAFF.getRole()) - || elem.get(6).equals(Mode.MANAGER.getRole()) - ? elem.get(3) - : elem.get(5), - elem -> new String[] { elem.get(5), elem.get(4), + || elem.get(6).equals(Mode.MANAGER.getRole()) ? elem.get(3).toLowerCase() + : elem.get(5).toLowerCase(), + elem -> new String[] { elem.get(5).toLowerCase(), elem.get(4), elem.get(6).equals(Mode.STAFF.getRole()) || elem.get(6).equals(Mode.MANAGER.getRole()) ? "." : elem.get(3) }, (elem1, elem2) -> elem1)); @@ -181,15 +202,15 @@ Integer extServerSid = extServer.getSid(); // load all users from DB which are present in the output file // map of user login -> user - Map allExistingParsedUsers = userManagementService + ConcurrentMap allExistingParsedUsers = userManagementService .findByPropertyValues(User.class, "login", allParsedUserMapping.keySet()).parallelStream() - .collect(Collectors.toConcurrentMap(User::getLogin, u -> u)); + .collect(Collectors.toConcurrentMap(u -> u.getLogin().toLowerCase(), u -> u)); logger.info(allExistingParsedUsers.size() + " users already exist"); // load all ext users from DB which are present in the output file // map of user login -> extUser - Map allExistingParsedExtUsers = userManagementService + ConcurrentMap allExistingParsedExtUsers = userManagementService .findByPropertyValues(ExtUserUseridMap.class, "extUsername", allExistingParsedUsers.keySet()) .parallelStream().filter(e -> e.getExtServer().getSid().equals(extServerSid)) .collect(Collectors.toConcurrentMap(ExtUserUseridMap::getExtUsername, e -> e)); @@ -207,7 +228,7 @@ // load all organisations from DB which are present in the output file, by code // map of code -> organisation - Map allExistingParsedCourses = userManagementService + ConcurrentMap allExistingParsedCourses = userManagementService .findByPropertyValues(Organisation.class, "code", allParsedCourseMapping.keySet()) .parallelStream().collect(Collectors.toConcurrentMap(Organisation::getCode, o -> o)); @@ -226,13 +247,13 @@ logger.info(allParsedSubcourseMapping.size() + " subcourses already exist"); // map of course ID -> subcourse code -> subcourse - Map> allExistingParsedCoursesAndSubcourses = allExistingParsedSubcourses + ConcurrentMap> allExistingParsedCoursesAndSubcourses = allExistingParsedSubcourses .parallelStream().filter(o -> o.getParentOrganisation() != null) .collect(Collectors.groupingByConcurrent(o -> o.getParentOrganisation().getOrganisationId(), Collectors.toConcurrentMap(Organisation::getCode, o -> o))); // load all ext courses and subcourses from DB which are present in the output file - Map allExistingParsedExtCourses = userManagementService + ConcurrentMap allExistingParsedExtCourses = userManagementService .findByPropertyValues(ExtCourseClassMap.class, "classid", Stream // merge IDs of organisations and suborganisations .concat(allExistingParsedCourses.values().stream(), @@ -247,7 +268,7 @@ .collect(Collectors.mapping(UserOrganisation::getUser, Collectors.toSet())); // map lines into corresponding roles - Map>> linesByMode = allLines.stream() + ConcurrentMap>> linesByMode = allLines.stream() .collect(Collectors.groupingByConcurrent(row -> row.get(6))); for (String role : Mode.getAllRoles()) { @@ -283,21 +304,55 @@ if (mode == Mode.MANAGER) { // map of course code -> user logins Map> mappings = lines.stream() - .collect(Collectors.groupingBy(elem -> elem.get(0), LinkedHashMap::new, + .collect(Collectors.groupingByConcurrent(elem -> elem.get(0), ConcurrentHashMap::new, Collectors.mapping(elem -> elem.get(3), Collectors.toList()))); AtomicInteger mappingsProcessed = new AtomicInteger(); logger.info("Processing manager courses and assigments"); - for (Entry courseEntry : allParsedCourseMapping.entrySet()) { - Organisation course = getCourse(courseEntry.getKey(), courseEntry.getValue(), extServer, - creatorId, rootOrganisation, allExistingParsedCourses, allExistingParsedExtCourses); + Collection>> spliterators = splitCollection( + allParsedCourseMapping.entrySet()); + List> futures = new ArrayList<>(spliterators.size()); - assignManagers(course, creatorId, mappings, allParsedUsers, userIDs, allExistingRoles, - allExistingParsedUsers, mappingsProcessed); + for (Spliterator> spliterator : spliterators) { + logger.info("Managers processing split count: " + spliterator.estimateSize()); - logger.info("Processed " + mappingsProcessed.get() + " entries"); + futures.add(executor.submit(new Callable() { + @Override + public Integer call() throws Exception { + try { + HibernateSessionManager.openSession(); + boolean elementsRemaining = true; + do { + elementsRemaining = spliterator.tryAdvance(courseEntry -> { + logger.info("Processing course: " + courseEntry.getValue()); + + try { + Organisation course = getCourse(courseEntry.getKey(), + courseEntry.getValue(), extServer, creatorId, + rootOrganisation, allExistingParsedCourses, + allExistingParsedExtCourses); + assignManagers(course, creatorId, mappings, allParsedUsers, userIDs, + allExistingRoles, allExistingParsedUsers, + mappingsProcessed); + + logger.info("Processed " + mappingsProcessed.get() + " entries"); + } catch (UserInfoValidationException | InterruptedException + | ExecutionException e) { + logger.error("Error while processing managers", e); + } + }); + } while (elementsRemaining); + } finally { + HibernateSessionManager.closeSession(); + } + return null; + } + })); } + for (Future future : futures) { + future.get(); + } logger.info("Processing manager role finished"); // END OF GROUP MANAGER PROCESSING @@ -307,9 +362,9 @@ // START OF LEARNER / STAFF PROCESSING // map of course code -> subcourse code -> user logins - Map>> mappings = lines.stream() - .collect(Collectors.groupingBy(elem -> elem.get(0), LinkedHashMap::new, - Collectors.groupingBy(elem -> elem.get(2), LinkedHashMap::new, + ConcurrentMap>> mappings = lines.stream() + .collect(Collectors.groupingByConcurrent(elem -> elem.get(0), ConcurrentHashMap::new, + Collectors.groupingByConcurrent(elem -> elem.get(2), ConcurrentHashMap::new, Collectors.mapping(elem -> mode == Mode.STAFF ? elem.get(3) : elem.get(5), Collectors.toList())))); @@ -353,11 +408,7 @@ } catch (Exception e) { logger.error("Error while provisioning SP enrolments", e); } finally { - try { - HibernateSessionManager.closeSession(); - } catch (Exception e) { - logger.error("Error while closing Hibernate session", e); - } + HibernateSessionManager.closeSession(); } }).start(); @@ -367,61 +418,93 @@ private Set createUsers(ExtServer extServer, Integer creatorId, Map users, Map userIDs, Map allExistingUsers, Map allExistingExtUsers) - throws UserInfoValidationException, UserInfoFetchException { - Set allUsersParsed = new HashSet<>(); + throws UserInfoValidationException, UserInfoFetchException, InterruptedException, ExecutionException { + Set allUsersParsed = ConcurrentHashMap.newKeySet(); logger.info("Creating users"); - for (Entry userEntry : users.entrySet()) { - // email servers as login - String login = userEntry.getKey(); - User user = allExistingUsers.get(login); - if (user == null) { - String salt = HashUtil.salt(); - String password = HashUtil.sha256(RandomPasswordGenerator.nextPassword(10), salt); - String email = userEntry.getValue()[0]; - String firstName = userEntry.getValue()[1]; - String lastName = userEntry.getValue()[2]; - ExtUserUseridMap userMap = integrationService.getImplicitExtUserUseridMap(extServer, login, password, - salt, firstName, lastName, email); - user = userMap.getUser(); + Collection>> spliterators = splitCollection(users.entrySet()); + List> futures = new ArrayList<>(spliterators.size()); - allExistingUsers.put(login, user); - allExistingExtUsers.put(login, userMap); + for (Spliterator> spliterator : spliterators) { + logger.info("Users processing split count: " + spliterator.estimateSize()); - String message = "User created with login \"" + login + "\" and ID " + user.getUserId(); - logger.info(message); - logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, null, - "SPEnrolment: " + message); - } else { - ExtUserUseridMap userMap = allExistingExtUsers.get(login); - if (userMap == null) { - userMap = new ExtUserUseridMap(); - userMap.setExtServer(extServer); - userMap.setUser(user); - userMap.setExtUsername(login); - userManagementService.save(userMap); + futures.add(executor.submit(new Callable() { + @Override + public Integer call() throws Exception { + try { + HibernateSessionManager.openSession(); + boolean elementsRemaining = true; + do { + elementsRemaining = spliterator.tryAdvance(userEntry -> { - allExistingExtUsers.put(login, userMap); + // email servers as login + String login = userEntry.getKey(); + User user = allExistingUsers.get(login.toLowerCase()); + if (user == null) { + String salt = HashUtil.salt(); + String password = HashUtil.sha256(RandomPasswordGenerator.nextPassword(10), salt); - String message = "External user created for existing user with login \"" + login + "\" and ID " - + user.getUserId(); - logger.info(message); - logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, null, - "SPEnrolment: " + message); - } - if (user.getDisabledFlag()) { - // re-enable user who was disabled before - user.setDisabledFlag(false); - userManagementService.save(user); - } - } + String email = userEntry.getValue()[0]; + String firstName = userEntry.getValue()[1]; + String lastName = userEntry.getValue()[2]; + ExtUserUseridMap userMap = null; + try { + userMap = integrationService.getImplicitExtUserUseridMap(extServer, login, + password, salt, firstName, lastName, email); + } catch (UserInfoValidationException e) { + logger.error("Error while getting/creating external user: " + login, e); + return; + } + user = userMap.getUser(); - // fill data for later usage - userIDs.put(login, user.getUserId()); - // add user to a collection of all users in the parsed file - allUsersParsed.add(user); - } + allExistingUsers.put(login, user); + allExistingExtUsers.put(login, userMap); + String message = "User created with login \"" + login + "\" and ID " + + user.getUserId(); + logger.info(message); + logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, null, + "SPEnrolment: " + message); + } else { + ExtUserUseridMap userMap = allExistingExtUsers.get(login); + if (userMap == null) { + userMap = new ExtUserUseridMap(); + userMap.setExtServer(extServer); + userMap.setUser(user); + userMap.setExtUsername(login); + userManagementService.save(userMap); + + allExistingExtUsers.put(login, userMap); + + String message = "External user created for existing user with login \"" + login + + "\" and ID " + user.getUserId(); + logger.info(message); + logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, + null, "SPEnrolment: " + message); + } + if (user.getDisabledFlag()) { + // re-enable user who was disabled before + user.setDisabledFlag(false); + userManagementService.save(user); + } + } + + // fill data for later usage + userIDs.put(login, user.getUserId()); + // add user to a collection of all users in the parsed file + allUsersParsed.add(user); + }); + } while (elementsRemaining); + } finally { + HibernateSessionManager.closeSession(); + } + return null; + } + })); + } + for (Future future : futures) { + future.get(); + } return allUsersParsed; } @@ -447,6 +530,7 @@ logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, null, "SPEnrolment: " + message); } else { + course = userManagementService.getOrganisationById(course.getOrganisationId()); String name = course.getName(); ExtCourseClassMap extOrgMap = allExistingParsedExtCourses.get(course.getOrganisationId()); @@ -470,172 +554,221 @@ } @SuppressWarnings("unchecked") - private void assignLearnersOrStaff(Organisation course, Map>> mappings, + private void assignLearnersOrStaff(Organisation course, Map>> mappings, ExtServer extServer, Integer creatorId, Set allUsersParsed, Map userIDs, Map>> allExistingRoles, Map> allExistingParsedCoursesAndSubcourses, Map allExistingParsedExtCourses, Map allExistingParsedUsers, - boolean isStaffMode, AtomicInteger mappingsProcessed) throws UserInfoValidationException { + boolean isStaffMode, AtomicInteger mappingsProcessed) + throws UserInfoValidationException, InterruptedException, ExecutionException { String courseCode = course.getCode(); Integer courseId = course.getOrganisationId(); ConcurrentMap existingSubcourses = allExistingParsedCoursesAndSubcourses.get(courseId); - Map nonProcessedSubcourses = existingSubcourses == null ? new HashMap<>() - : new HashMap<>(existingSubcourses); + Map nonProcessedSubcourses = existingSubcourses == null ? new ConcurrentHashMap<>() + : new ConcurrentHashMap<>(existingSubcourses); // go through each subcourse Map> subcourseMappings = mappings.get(courseCode); if (subcourseMappings != null) { - for (Entry> subcourseEntry : subcourseMappings.entrySet()) { - String subcourseCode = subcourseEntry.getKey(); - nonProcessedSubcourses.remove(subcourseCode); + Collection>>> spliterators = splitCollection( + subcourseMappings.entrySet()); + List> futures = new ArrayList<>(spliterators.size()); - Organisation subcourse = existingSubcourses == null ? null : existingSubcourses.get(subcourseCode); - // create subcourse - if (subcourse == null) { - ExtCourseClassMap extSubOrgMap = integrationService.createExtCourseClassMap(extServer, creatorId, - subcourseCode, subcourseCode, course.getOrganisationId().toString(), false); - subcourse = extSubOrgMap.getOrganisation(); - subcourse.setCode(subcourseCode); - userManagementService.save(subcourse); + for (Spliterator>> spliterator : spliterators) { + logger.info("Learners/staff processing split count: " + spliterator.estimateSize()); - if (existingSubcourses == null) { - existingSubcourses = new ConcurrentHashMap<>(); - allExistingParsedCoursesAndSubcourses.put(courseId, existingSubcourses); - } - existingSubcourses.put(subcourse.getCode(), subcourse); - allExistingParsedExtCourses.put(extSubOrgMap.getOrganisation().getOrganisationId(), extSubOrgMap); + futures.add(executor.submit(new Callable() { + @Override + public Integer call() throws Exception { + try { + HibernateSessionManager.openSession(); + boolean elementsRemaining = true; + do { + elementsRemaining = spliterator.tryAdvance(subcourseEntry -> { + try { + ConcurrentMap existingSubcoursesInternal = existingSubcourses; + String subcourseCode = subcourseEntry.getKey(); + nonProcessedSubcourses.remove(subcourseCode); - String message = "Subcourse created with code and name \"" + courseCode + "\" and ID " - + subcourse.getOrganisationId(); - logger.info(message); - logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, null, - "SPEnrolment: " + message); - } else { - String name = subcourse.getName(); + logger.info("Processing subcourse with code: " + subcourseCode); - ExtCourseClassMap extOrgMap = allExistingParsedExtCourses.get(subcourse.getOrganisationId()); - if (extOrgMap == null) { - extOrgMap = new ExtCourseClassMap(); - extOrgMap.setCourseid(name); - extOrgMap.setExtServer(extServer); - extOrgMap.setOrganisation(subcourse); - userManagementService.save(extOrgMap); + Organisation subcourse = existingSubcoursesInternal == null ? null + : existingSubcoursesInternal.get(subcourseCode); + // create subcourse + if (subcourse == null) { + ExtCourseClassMap extSubOrgMap; - String message = "External subcourse created for existing subcourse with code \"" - + subcourseCode + "\" and name \"" + name + "\" and ID " - + subcourse.getOrganisationId(); - logger.info(message); - logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, null, - "SPEnrolment: " + message); - } - } + extSubOrgMap = integrationService.createExtCourseClassMap(extServer, + creatorId, subcourseCode, subcourseCode, + course.getOrganisationId().toString(), false); - Integer subcourseId = subcourse.getOrganisationId(); + subcourse = extSubOrgMap.getOrganisation(); + subcourse.setCode(subcourseCode); + userManagementService.save(subcourse); - // get existing learners/staff members for given subcourse - Collection subcourseMonitorsOrLearners = userManagementService - .getUsersFromOrganisationByRole(subcourseId, isStaffMode ? Role.MONITOR : Role.LEARNER, true); - Collection subcourseUsers = new HashSet<>(subcourseMonitorsOrLearners); - if (isStaffMode) { - // make sure that staff has both monitor and author roles in subcourse, - // even if they are course managers in the parent organisations - // and they already have a monitor role in subcourse - Collection authors = userManagementService.getUsersFromOrganisationByRole(subcourseId, - Role.AUTHOR, true); - subcourseUsers.retainAll(authors); - } + if (existingSubcoursesInternal == null) { + existingSubcoursesInternal = new ConcurrentHashMap<>(); + allExistingParsedCoursesAndSubcourses.put(courseId, + existingSubcoursesInternal); + } + existingSubcoursesInternal.put(subcourse.getCode(), subcourse); + allExistingParsedExtCourses.put( + extSubOrgMap.getOrganisation().getOrganisationId(), extSubOrgMap); - // go through each user - for (String login : subcourseEntry.getValue()) { + String message = "Subcourse created with code and name \"" + courseCode + + "\" and ID " + subcourse.getOrganisationId(); + logger.info(message); + logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, + null, null, "SPEnrolment: " + message); + } else { + String name = subcourse.getName(); - logger.info("Processing \"" + login + "\""); + ExtCourseClassMap extOrgMap = allExistingParsedExtCourses + .get(subcourse.getOrganisationId()); + if (extOrgMap == null) { + extOrgMap = new ExtCourseClassMap(); + extOrgMap.setCourseid(name); + extOrgMap.setExtServer(extServer); + extOrgMap.setOrganisation(subcourse); + userManagementService.save(extOrgMap); - mappingsProcessed.incrementAndGet(); + String message = "External subcourse created for existing subcourse with code \"" + + subcourseCode + "\" and name \"" + name + "\" and ID " + + subcourse.getOrganisationId(); + logger.info(message); + logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, + null, null, "SPEnrolment: " + message); + } + } - // check if the user is already a learner/staff member in the subcourse - // if so, there is nothing to do - boolean userAlreadyAssigned = false; - Integer userId = userIDs.get(login); - Iterator subcourseUserIterator = subcourseUsers.iterator(); - while (subcourseUserIterator.hasNext()) { - User user = subcourseUserIterator.next(); - if (userId.equals(user.getUserId())) { - // IMPORTANT: if we found a matching existing learner/staff member, we get remove him from this collection - // so after this loop he does not get removed from subcourses - subcourseUserIterator.remove(); - subcourseMonitorsOrLearners.remove(user); - userAlreadyAssigned = true; - break; - } - } - if (userAlreadyAssigned) { - continue; - } + Integer subcourseId = subcourse.getOrganisationId(); - // the user is not a learner/staff member yet, so assign him the role and add him to lessons - Map> existingSubcoursesRoles = allExistingRoles.get(login); - Set existingSubcourseRoles = existingSubcoursesRoles == null ? null - : allExistingRoles.get(login).get(subcourseId); - if (existingSubcourseRoles == null) { - existingSubcourseRoles = new HashSet<>(); - } - if (isStaffMode) { - existingSubcourseRoles.add(Role.ROLE_AUTHOR); - existingSubcourseRoles.add(Role.ROLE_MONITOR); - } else { - existingSubcourseRoles.add(Role.ROLE_LEARNER); - } - User user = allExistingParsedUsers.get(login); - userManagementService.setRolesForUserOrganisation(user, subcourse, - existingSubcourseRoles.stream().map(String::valueOf).collect(Collectors.toList()), false); + // get existing learners/staff members for given subcourse + Collection subcourseMonitorsOrLearners = userManagementService + .getUsersFromOrganisationByRole(subcourseId, + isStaffMode ? Role.MONITOR : Role.LEARNER, true); + Set subcourseUsers = ConcurrentHashMap.newKeySet(); + subcourseUsers.addAll(subcourseMonitorsOrLearners); + if (isStaffMode) { + // make sure that staff has both monitor and author roles in subcourse, + // even if they are course managers in the parent organisations + // and they already have a monitor role in subcourse + Collection authors = userManagementService + .getUsersFromOrganisationByRole(subcourseId, Role.AUTHOR, true); + subcourseUsers.retainAll(authors); + } - for (Lesson lesson : lessonService.getLessonsByGroup(subcourseId)) { - if (isStaffMode) { - lessonService.addStaffMember(lesson.getLessonId(), userId); - } else { - lessonService.addLearner(lesson.getLessonId(), userId); + final Organisation finalSubcourse = subcourse; + + for (String login : subcourseEntry.getValue()) { + + logger.info("Processing \"" + login + "\""); + + mappingsProcessed.incrementAndGet(); + + // check if the user is already a learner/staff member in the subcourse + // if so, there is nothing to do + boolean userAlreadyAssigned = false; + Integer userId = userIDs.get(login); + Iterator subcourseUserIterator = subcourseUsers.iterator(); + while (subcourseUserIterator.hasNext()) { + User user = subcourseUserIterator.next(); + if (userId.equals(user.getUserId())) { + // IMPORTANT: if we found a matching existing learner/staff member, we get remove him from this collection + // so after this loop he does not get removed from subcourses + subcourseUserIterator.remove(); + subcourseMonitorsOrLearners.remove(user); + userAlreadyAssigned = true; + break; + } + } + if (userAlreadyAssigned) { + return; + } + + // the user is not a learner/staff member yet, so assign him the role and add him to lessons + Map> existingSubcoursesRoles = allExistingRoles + .get(login); + Set existingSubcourseRoles = existingSubcoursesRoles == null ? null + : allExistingRoles.get(login).get(subcourseId); + if (existingSubcourseRoles == null) { + existingSubcourseRoles = new HashSet<>(); + } + if (isStaffMode) { + existingSubcourseRoles.add(Role.ROLE_AUTHOR); + existingSubcourseRoles.add(Role.ROLE_MONITOR); + } else { + existingSubcourseRoles.add(Role.ROLE_LEARNER); + } + User user = allExistingParsedUsers.get(login.toLowerCase()); + userManagementService + .setRolesForUserOrganisation( + user, finalSubcourse, existingSubcourseRoles.stream() + .map(String::valueOf).collect(Collectors.toList()), + false); + + for (Lesson lesson : lessonService.getLessonsByGroup(subcourseId)) { + if (isStaffMode) { + lessonService.addStaffMember(lesson.getLessonId(), userId); + } else { + lessonService.addLearner(lesson.getLessonId(), userId); + } + } + + String message = (isStaffMode ? "Teacher" : "Learner") + " \"" + login + + "\" added to subcourse " + subcourseId + " and its lessons"; + logger.info(message); + logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, + null, null, "SPEnrolment: " + message); + } + + // user is a learner/staff member, but he should not; remove him role from subcourse, course and lessons + for (User user : subcourseMonitorsOrLearners) { + + boolean removedFromSubcourse = removeFromCourse(subcourse, user, + isStaffMode ? Mode.STAFF : Mode.LEARNER, allExistingRoles); + if (removedFromSubcourse) { + String message = (isStaffMode ? "Teacher" : "Learner") + " \"" + + user.getLogin() + "\" removed from subcourse " + subcourseId + + " and its lessons"; + logger.info(message); + logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, + null, null, "SPEnrolment: " + message); + } + } + } catch (UserInfoValidationException e) { + logger.error("Error while processing learners/staff", e); + } + }); + } while (elementsRemaining); + } finally { + HibernateSessionManager.closeSession(); } + return null; } + })); + } + for (Future future : futures) { + future.get(); + } - String message = (isStaffMode ? "Teacher" : "Learner") + " \"" + login + "\" added to subcourse " - + subcourseId + " and its lessons"; - logger.info(message); - logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, null, - "SPEnrolment: " + message); - } - - // user is a learner/staff member, but he should not; remove him role from subcourse, course and lessons + for (Organisation subcourse : nonProcessedSubcourses.values()) { + // get all learners/staff members for given subcourse and remove them + Collection subcourseMonitorsOrLearners = userManagementService.getUsersFromOrganisationByRole( + subcourse.getOrganisationId(), isStaffMode ? Role.MONITOR : Role.LEARNER, true); for (User user : subcourseMonitorsOrLearners) { boolean removedFromSubcourse = removeFromCourse(subcourse, user, isStaffMode ? Mode.STAFF : Mode.LEARNER, allExistingRoles); if (removedFromSubcourse) { String message = (isStaffMode ? "Teacher" : "Learner") + " \"" + user.getLogin() - + "\" removed from subcourse " + subcourseId + " and its lessons"; + + "\" removed from subcourse " + subcourse.getOrganisationId() + " and its lessons"; logger.info(message); logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, null, "SPEnrolment: " + message); } } } } - - for (Organisation subcourse : nonProcessedSubcourses.values()) { - // get all learners/staff members for given subcourse and remove them - Collection subcourseMonitorsOrLearners = userManagementService.getUsersFromOrganisationByRole( - subcourse.getOrganisationId(), isStaffMode ? Role.MONITOR : Role.LEARNER, true); - for (User user : subcourseMonitorsOrLearners) { - - boolean removedFromSubcourse = removeFromCourse(subcourse, user, - isStaffMode ? Mode.STAFF : Mode.LEARNER, allExistingRoles); - if (removedFromSubcourse) { - String message = (isStaffMode ? "Teacher" : "Learner") + " \"" + user.getLogin() - + "\" removed from subcourse " + subcourse.getOrganisationId() + " and its lessons"; - logger.info(message); - logEventService.logEvent(LogEvent.TYPE_USER_ORG_ADMIN, creatorId, null, null, null, - "SPEnrolment: " + message); - } - } - } } private boolean removeFromCourse(Organisation course, User user, Mode mode, @@ -699,7 +832,8 @@ private void assignManagers(Organisation course, Integer creatorId, Map> mappings, Set allUsersParsed, Map userIDs, Map>> allExistingRoles, Map allExistingParsedUsers, - AtomicInteger mappingsProcessed) throws UserInfoValidationException { + AtomicInteger mappingsProcessed) + throws UserInfoValidationException, InterruptedException, ExecutionException { String courseCode = course.getCode(); Integer courseId = course.getOrganisationId(); @@ -733,11 +867,12 @@ Set existingCourseRoles = existingCoursesRoles == null ? null : allExistingRoles.get(login).get(courseId); if (existingCourseRoles == null) { - existingCourseRoles = new HashSet<>(); + existingCourseRoles = ConcurrentHashMap.newKeySet(); } existingCourseRoles.add(Role.ROLE_GROUP_MANAGER); - User user = allExistingParsedUsers.get(login); + User user = allExistingParsedUsers.get(login.toLowerCase()); + userManagementService.setRolesForUserOrganisation(user, course, existingCourseRoles.stream().map(String::valueOf).collect(Collectors.toList()), true); @@ -762,6 +897,26 @@ } } + /** + * Splits collection into as many spliterators as there are threads + */ + private Collection> splitCollection(Collection collection) { + // if there is only 100 entries, do it in one thread + int threadCount = collection.size() < 100 ? 1 : this.threadCount; + + LinkedList> spliterators = new LinkedList<>(); + spliterators.add(collection.spliterator()); + for (int threadIndex = 1; threadIndex < threadCount; threadIndex++) { + Spliterator spliterator = spliterators.removeFirst(); + spliterators.add(spliterator); + Spliterator anotherSpliterator = spliterator.trySplit(); + if (anotherSpliterator != null) { + spliterators.add(anotherSpliterator); + } + } + return spliterators; + } + @Override public void init() throws ServletException { lessonService = (ILessonService) WebApplicationContextUtils