Feat : Added pagination logic to migrations API

controllers/users.js

@@ -857,20 +857,16 @@ async function usersPatchHandler(req, res) {
  }
 }
 const migrations = async (req, res) => {
- const { action } = req.query;
- if (action !== "adds-github-id") {
- return res.status(400).json({
- message: "Invalid action type",
- });
- }
+ const { skip = 0, limit } = req.query;
 
  try {
- const result = await addGithubUserId();
+ const result = await addGithubUserId(parseInt(skip), parseInt(limit));
  return res.status(200).json({
  message: "Result of migration",
  data: result,
  });
  } catch (error) {
+ logger.error(`Internal Server Error: ${error}`);
  return res.boom.badImplementation(INTERNAL_SERVER_ERROR);
  }
 };

models/users.js

@@ -868,7 +868,7 @@ const getNonNickNameSyncedUsers = async () => {
  throw err;
  }
 };
-const addGithubUserId = async () => {
+const addGithubUserId = async (skip, limit) => {
  try {
  const usersNotFound = [];
  let countUserFound = 0;
@@ -882,45 +882,43 @@ const addGithubUserId = async () => {
  "Basic " + btoa(`${config.get("githubOauth.clientId")}:${config.get("githubOauth.clientSecret")}`),
  },
  };
- const usersSnapshot = await firestore.collection("users").get();
- const totalUsers = usersSnapshot.docs.length;
- const batchCount = Math.ceil(totalUsers / 500);
+ const usersSnapshot = await firestore
+ .collection("users")
+ .limit(limit)
+ .offset(skip * limit)
+ .get();
  // Create batch write operations for each batch of documents
- for (let i = 0; i < batchCount; i++) {
- const batchWrite = firestore.batch();
- const batchWrites = [];
- for (const userDoc of usersSnapshot.docs.slice(i * 500, (i + 1) * 500)) {
- if (userDoc.data().github_user_id) continue;
- const githubUsername = userDoc.data().github_id;
- const username = userDoc.data().username;
- const userId = userDoc.id;
- batchWrite.update(userDoc.ref, { github_user_id: null });
-
- const getUserDetails = fetch(`https://api.github.com/users/${githubUsername}`, requestOptions)
- .then((response) => {
- if (!response.ok) {
- throw new Error("Network response was not ok");
- }
- return response.json();
- })
- .then((data) => {
- const githubUserId = data.id;
- batchWrite.update(userDoc.ref, { github_user_id: `${githubUserId}` });
- countUserFound++;
- })
- .catch((error) => {
- countUserNotFound++;
- const invalidUsers = { userId, username, githubUsername };
- usersNotFound.push(invalidUsers);
- logger.error("An error occurred at fetch:", error);
- });
- batchWrites.push(getUserDetails);
- }
- await Promise.all(batchWrites);
- await batchWrite.commit();
+ const batchWrite = firestore.batch();
+ const batchWrites = [];
+ for (const userDoc of usersSnapshot.docs) {
+ if (userDoc.data().github_user_id) continue;
+ const githubUsername = userDoc.data().github_id;
+ const username = userDoc.data().username;
+ const userId = userDoc.id;
+ const getUserDetails = fetch(`https://api.github.com/users/${githubUsername}`, requestOptions)
+ .then((response) => {
+ if (!response.ok) {
+ throw new Error("Network response was not ok");
+ }
+ return response.json();
+ })
+ .then((data) => {
+ const githubUserId = data.id;
+ batchWrite.update(userDoc.ref, { github_user_id: `${githubUserId}` });
+ countUserFound++;
+ })
+ .catch((error) => {
+ countUserNotFound++;
+ const invalidUsers = { userId, username, githubUsername };
+ usersNotFound.push(invalidUsers);
+ logger.error("An error occurred at fetch:", error);
+ });
+ batchWrites.push(getUserDetails);
  }
+ await Promise.all(batchWrites);
+ await batchWrite.commit();
  return {
- totalUsers: totalUsers,
+ totalUsers: usersSnapshot.docs.length,
  usersUpdated: countUserFound,
  usersNotUpdated: countUserNotFound,
  invalidUsersDetails: usersNotFound,

test/unit/models/users.test.js

@@ -493,7 +493,7 @@ describe("users", function () {
  it("API should not be accessible to any regular user", function (done) {
  chai
  .request(app)
- .post("/users/migrations?action=adds-github-id")
+ .post("/users/migrations?action=adds-github-id&skip=0&limit=10")
  .set("cookie", `${cookieName}=${userToken}`)
  .send()
  .end((err, res) => {
@@ -523,7 +523,7 @@ describe("users", function () {
 
  const res = await chai
  .request(app)
- .post("/users/migrations?action=adds-github-id")
+ .post("/users/migrations?action=adds-github-id&skip=0&limit=10")
  .set("cookie", `${cookieName}=${superUserToken}`)
  .send();