Queues with Kue.js, Node.js and Redis

Reference these links:

https://medium.com/@NorbertdeLangen/communicating-between-nodejs-processes-4e68be42b917 https://medium.com/@niratattri/workers-and-node-kue-it-up-1c1215d9bddf https://github.com/Automattic/kue

tl;dr

const { fork } = require("child_process");

const kue = require("kue");
kue.app.listen(3050);
const port = process.env.REDIS_PORT ? process.env.REDIS_PORT : "6379";
const host = process.env.REDIS_HOST ? process.env.REDIS_HOST : "127.0.0.1";

let queue = kue.createQueue({
  redis: {
    host: host,
    port: port,
  },
});

queue.process("build", 1, (job, done) => {
  run(job, done);
});

const run = async (job, done) => {
  try {
    const { data } = job.data;
    // process is a forked process
    const compute = fork("./processes/buildWeb.js");
    compute.send(data);

    compute.on("message", (_) => {
      return done();
    });
  } catch (err) {
    console.error(err);
    return done(new Error(JSON.stringify(err)));
  }
};

module.exports = (app) => {
  app.post("/", async function (req, res) {
    try {
      // Create a fork for a process
      const buildJob = queue
        .create("build", {
          // Job Type
          project: project, // Job Data
          data: req.body,
        })
        .removeOnComplete(true) // REMOVE THE JOB FROM THE QUEUE ONCE IT'S COMPLETED
        .attempts(5) // The maximum number of retries you want the job to have
        .backoff({ delay: 60 * 1000, type: "exponential" }) // Time between retries. Read docs.
        .save(); // PERSIST THE DAMN JOB LOL

      buildJob.on("failed", function (errorMessage) {
        console.log("Job failed");
        let error = JSON.parse(errorMessage);
        // error now contains the object passed from the worker when the job failed
        console.log(error); // Check it out for yourself
        // call pagerduty or whatever jazz you wanna do in case of failure
      });

      res.status(200).send("Building " + project);
    } catch (err) {
      return res.status(500).send("Failed");
    }
  });
};

// buildWeb.js
const run = async (data) => {
  try {
    setTimeout(() => {
      process.send("Done");
    }, 3000);
  } catch (err) {
    console.error(err);
  }
};

process.on("message", (data) => {
  run(data);
});