Building a Node Express proxy API for Elasticsearch

It is a best practice to hide your Elasticsearch instance behind an API or a Proxy. This is obviously to control non-Admin access from you end users but also to provide clean end-points to your custom client apps.

In this article I will take a fictitious business case that requires use of Elasticsearch index data and I will show how to build a Node Express API that acts as a proxy between your client app and Elasticsearch.

Use cases covered:

1. Secure proxy API to Elasticcearch
2. Filter outgoing data by user

Interesting patterns covered:

1. User Authentication & Authorization
2. Controlling which indexes can be searched
3. Logging and log file formatting to support FileBeat import into Elasticsearch for Kibana use
4. Use of Swagger wrapper to get basic documentation and a tester page
5. Use of DOTENV to handle environments
6. Use of Node Express MiddleWare to intercept calls

Before getting any further:

• GitLab repo here
• NodeJS > 10
• I’m using Express 4.16.3
• Elasticsearch client for node
• I will be using MS Visual Code as IDE

Setting up your project:

• Start with a command window create a new folder calling it elasticsearch_proxyapi
• npm init (assuming you know how to use this and that entry file is the default index.js)
• Now install the following npm packages:
• npm i express elasticsearch winston morgan @types/morgan @elastic/ecs-morgan-format @elastic/ecs-winston-format winston-daily-rotate-file
• With those installed, our packages.json file should have now been populated under your project directory
• Let’s do some more npm installs that we will use:
• npm i cors dotenv body-parser swagger-ui-express

Now we start coding:

• In Visual Code, create a new .env.development file under your root directory and paste the following. I will explain later what the env variables mean.

API_VESION='1.0'
ES_SERVER_URL="http://localhost:9200"
ES_RASTER='data_rasters'
ES_ACCESS='data_access'
LOG_LEVEL="debug"
DOMAIN='HOME'
SERVER_PORT=3000

• In Visual Code, create an index.js file under your root directory and we start coding:

//#region imports and requires
const environment = (process.env.NODE_ENV === 'development') ? 'development' : 'production'
require('dotenv').config({ path: `.env.${environment}` })

const express = require('express')

const bodyParser = require('body-parser')
const { promisify } = require('util') //The util.promisify() method defines in utilities module of Node.js standard library. It is basically used to convert a method that returns responses using a callback function to return responses in a promise object

const cors = require('cors');
const morgan = require('morgan');
const ecsFormat = require('@elastic/ecs-morgan-format') //Formats the logs using Elastic Common Schema

const _ = require('lodash');

const swaggerUi = require('swagger-ui-express');
const swaggerDocument = require(__dirname + '/config/swagger.json');

const helperES = require('./helperES')
const winstonLogger = require('./helperWinstonLogger')

const helperAMMiddleWare = require('./helperAMMiddleWare')

//#endregion imports and requires
// In case your API is not installed on same location as your elasticsearch instance
const headers = {
  "Content-Type": "application/json",
  "Access-Control-Allow-Origin": "*",
  "Access-Control-Allow-Headers": "Content-Type",
  "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTION"
}

Now get the Node Express app going and the Middleware:

const app = express()

console.log('RUNNING IN MODE: ' + environment)

//#region MiddleWares
app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument));

//add other middleware
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));

// Attach the custom Authorization Management middleware before starting the express app
app.use(helperAMMiddleWare)

app.use(
  morgan(ecsFormat(), {
    skip: function(req, res) {
      return res.statusCode < 400;
    },
    stream: winstonLogger.stream // process.stderr //winstonLogger.stream
  })
);

app.use(
  morgan(ecsFormat(), {
    skip: function(req, res) {
      return res.statusCode >= 400;
    },
    stream: winstonLogger.stream // process.stdout //winstonLogger.stream
  })
);
//#endregion MiddleWares

You may have noticed some custom js files. We’ll fill those in shortly. For now we finish the rest of the index.js file with some basic endpoints:

//#region API endpoints

app.get('/ping', async (req, res) => {
  try{

    res.set(headers)
    res.status(200).send(
      {
        body: { status: "success", result:  "Reply from Proxy API ver: " + (process.env.API_VESION === null ? 'none' : process.env.API_VESION) }
      }
    );

  }
  catch (err) {
    winstonLogger.error('Ping error:' + err)
    res.status(500).send(err);
}
});

app.get('/getcurrentuserinfo', async (req, res) => {
  try{

    res.set(headers)
    res.status(200).send(
      {
        body: { status: "success", result:  helperAMMiddleWare.userInfo }
      }
    );
    //res.send(helperAMMiddleWare.userInfo)
  }
  catch (err) {
    winstonLogger.error('getcurrentuserinfo error:' + err)
    res.status(500).send(err);
}
});

app.post('/:indexId/_search', async (req, res) => {
  try{

    if (!req.params){

      res.set(headers)
      res.status(400).send(
        {
        body: { status: "failure", result:  'A index name or pattern needs to be specified in the URI <baseuri>/indexname/_search' }
      });
    }
    else {

      let indexName = req.params["indexId"]
      //TODO: check request body !!!
      abc = await helperES.searchIndexAsync(indexName, req.body, helperAMMiddleWare.userCategoriesArray)
      .then( ret => {
        winstonLogger.debug('Retrieved ES Entries')
        winstonLogger.debug(ret)
        res.set(headers)
        res.status(200).send(
          {
            body: { status: "success", result:  ret }
          }
        )
        //res.send(JSON.stringify(ret))
      })
      .catch(err => {
        winstonLogger.error('Get ES Entries failed: ' + err)
      });

    }

  }
  catch (err) {
    winstonLogger.error('_Search error:' + err)
    res.status(500).send(err);
}
});

//#endregion API endpoints


//start the Node server
const startServer = async () => {

  const port = process.env.SERVER_PORT || 3000
  await promisify(app.listen).bind(app)(port)
  console.log(`Listening on port ${port}`)
}

startServer()

Notice how the /:indexId/_search end point accepts an index name as part of the URL.

• Now create a helperAMMiddleWare.js under the root folder:

const helperES = require('./helperES')
const winstonLogger = require('./helperWinstonLogger')

module.exports = async (req, res, next) => {
  try {

    if (!req.headers.user){
      throw new Error('Username is missing from the request header. Cannot proceed')
    }

    winstonLogger.debug(req.path + ' end point invoked by user: ' + req.headers.user)
    winstonLogger.debug('Authorization module invoked..')

    const userAccess = await helperES.client.get({
        index: process.env.ES_ACCESS,
        id: req.headers.user
      }, 
      {
          ignore: [404],
          maxRetries: 3
      }
    )
    
    var categoriesArray = []
    if (userAccess && userAccess.body.found) {
        categoriesArray = userAccess.body._source.dataaccess
        if (categoriesArray.lenght == 0) throw new Error('No data categories have been granted for User: ' + req.headers.user)
    }
    else throw new Error('User: ' + req.headers.user + ' is not granted access to any data!')
    
    winstonLogger.debug('User: ' + req.headers.user + ' has access to following data categories: ' + categoriesArray.join())

    module.exports.userCategoriesArray = categoriesArray
    module.exports.user = req.headers.user
    module.exports.userInfo = userAccess.body._source

    
    next()

  } catch (error) {
    next(error.message)
  }
}

Note how easily we’re consulting an elasticsearch index for the existence of an entry with this user as document id. When found, we build an array of data categories this user is allowed to access (fictitious use case to showcase elasticsearch post_filter capability)

Now we create a helperES.js under the root folder:

const environment = (process.env.NODE_ENV === 'development') ? 'development' : 'production'
require('dotenv').config({ path: `.env.${environment}` }) ////require('dotenv').config({ path: `.env.${process.env.NODE_ENV}` })

const winstonLogger = require('./helperWinstonLogger')

require('array.prototype.flatmap').shim()

const { Client } = require('@elastic/elasticsearch')

const client = new Client({
  node: process.env.ES_SERVER_URL, //'https://localhost:9200'
  auth: {
    username: 'elastic',
    password: 'changeme' //process.env.ESPWD ||| 
  }
})


module.exports =  {
    
    client,

    // Async Function
    searchIndexAsync: async function(indexname, reqBody, userCategoriesArray) {
        try{

            var modifiedreqBody = reqBody

            const post_filter = {terms: { "properties.location.categories.keyword": userCategoriesArray, "boost": 1.0 }} // *** Authorization filtering enforcment

            if (!modifiedreqBody.query ) modifiedreqBody.query = {"match_all": {}}

            delete modifiedreqBody["post_filter"] //strip the query from any existing post_filter

            if (userCategoriesArray[0] != 'All'){ //***************  GOD MODE DOES NOT GET APPLIED A POST FILTER
                modifiedreqBody["post_filter"] = post_filter
            }

            winstonLogger.debug(modifiedreqBody)

            const { body } = await client.search({ 
                index: indexname, 
                body: modifiedreqBody
            }, 
            {
                ignore: [404],
                maxRetries: 3
              }
            )

            const res = body.hits.hits.map(e => ({ _id: e._id, ...e._source }))
        
            winstonLogger.debug(res)

            return body //return res eventually
        }

        catch (err) {
            winstonLogger.error(err)
        }

    },
};

Notice how we append a post_filter to the query request to limit the result to data categories user is allowed to access.

• Now create a new helperWinstonLogger.js file:

const environment = (process.env.NODE_ENV === 'development') ? 'development' : 'production'
require('dotenv').config({ path: `.env.${environment}` }) ////require('dotenv').config({ path: `.env.${process.env.NODE_ENV}` })

const winston = require('winston');
const ecsWinstonFormat = require('@elastic/ecs-winston-format')

require('winston-daily-rotate-file');

const levels = {
  error: 0,
  warning: 1,
  info: 2,
  http: 3,
  debug: 4,
}

const level = () => {
  const isDevelopment = environment === 'development'
  return isDevelopment ? 'debug' : 'info'
}

const colors = {
  error: 'red',
  warn: 'yellow',
  info: 'green',
  http: 'magenta',
  debug: 'white',
}

winston.addColors(colors)

const transports = [
  new winston.transports.Console(),
  new winston.transports.DailyRotateFile({
    filename: 'logs/error-%DATE%.log',
    datePattern: 'YYYY-MM-DD',
    zippedArchive: true,
    maxSize: '20m',
    maxFiles: '14d',
    level: 'error',
  }),
  new winston.transports.DailyRotateFile({
    filename: 'logs/all-%DATE%.log',
    datePattern: 'YYYY-MM-DD',
    zippedArchive: true,
    maxSize: '20m',
    maxFiles: '14d',
    level: level(),
  })
]

const winstonLogger = winston.createLogger({
  level: level(),
  levels,
  format: ecsWinstonFormat(),
  transports: transports,
})

winstonLogger.stream = {
  write: function(message, encoding){
    winstonLogger.info(message);
  }
};

module.exports = winstonLogger

• Ensure your scripts session of the package.json has:

"scripts": {
    "start": "node .",
    "test": "standard"
  }

• You can now run ‘npm run start’ from the command line and your API will be listening on port 3000
• To access the Swagger UI documentation simply navigate to http://localhost:3000/api-docs/ and you should see:

• Finally, the logs generated are compatible with Elasticsearch, i.e. comply with ECS schema so you can point a FileBeat instance to them and ingest in Elasticsearch for view with Kibana dashboards (topic for another article).