Home |  Services |  Side Projects |  Posts (archive) |  Miles Davenport |  Career
lambda |  algorithms |  good devops |  bad devops
i'm so glad I chose this career |  the Konkan railway “Engineer message” |  Knowledge |  avoid |  RSS

Debugging Java 8 Streams

The peek() method is useful for debugging Java 8 stream(s).

This example doesn’t need an explicit peek(), but it shows how useful the peek method is for debugging:

package com.chocksaway;

import java.util.List;
import java.util.stream.Collectors;

/**
 * Author milesd on 31/12/2017.
 */
public class FilterNumbers {
    private List<Integer> numberList;

    public FilterNumbers(List<Integer> numberList) {
        this.numberList = numberList;
    }

    public List<Integer> process() {
        return numberList.stream()
            .filter(each -> each > 15)

            // peek for the first filter
            .peek(each -> System.out.println("Debug 001:\t " + each))

            .filter(each -> each % 3 == 0)

            // peek for the second filter
            .peek(each -> System.out.println("Debug 002:\t " + each))

            .collect(Collectors.toList());
    }
}
Corresponding Unit Test:
import com.chocksaway.FilterNumbers;
import org.junit.Test;

import java.util.ArrayList;
import java.util.List;

import static junit.framework.TestCase.assertTrue;

/**
 * Author milesd on 31/12/2017.
 */
public class TestFilterNumbers {
    @Test
    public void peekAtLambda() {
        List<Integer> numbers = new ArrayList<>(List.of(12, 13, 14, 15, 20, 21, 24));
        FilterNumbers filterGreaterFifteenDivisibleByThree = new FilterNumbers(numbers);

        List<Integer> result = filterGreaterFifteenDivisibleByThree.process();

        assertTrue(result.contains(21));
        assertTrue(result.contains(24));
    }
}
Running the Unit Test

The debug output:

Debug 001:	 20
Debug 001:	 21
Debug 002:	 21
Debug 001:	 24
Debug 002:	 24

peek() shows the Debug 001, and Debug 002 for the first and second filter second statements.

»


Node and AWS API Gateway

Happy New Year :D

You would think that integrating a call from a Node app into an AWS API Gateway would be fairly easy.

It wasn’t.

A number of things prevented me from working efficiently:

  • Context switching - I need time to learn, play, gain expertise, fine-tune, and pass on the knowledge.

    This hasn’t happened. The constant problems with a third-party platform prevent this focus on a single task.

  • Different / later version of Node (which use things like async)

  • Not being able to develop locally.

  • Not having the IAM roles / permissions to work.

Over the break, I wanted to move forward, and break-the-back of this task

I went back to basics, and grabbed my notes from the February 2017 meetup, where I had presented on using the AWS API Gateway.

Miles Davenport

I used the sample PetStore (provided by the API Gateway), and created an API key, and IAM Role which was associated to the PetStore, and Usage plan

I started off by using just the API key to test an endpoint with:

GET /int//pets/1 HTTP/1.1
Content-Type: application/json
x-api-key: [snip]
Host: [snip]
Connection: close
User-Agent: Paw/3.1.4 (Macintosh; OS X/10.13.2) GCDHTTPRequest


The response is here:

HTTP/1.1 200 OK
Date: Sun, 31 Dec 2017 16:12:33 GMT
Content-Type: application/json
Content-Length: 49
Connection: close
x-amzn-RequestId: [snip]
Access-Control-Allow-Origin: *
X-Amzn-Trace-Id: Root= [snip]

{
  "id": 1,
  "type": "dog",
  "price": 249.99
}

Integrating with Node

The next step was integrating with Node v6.10.3.

I wanted to use https://www.npmjs.com/package/aws-api-gateway-client.

I have developed the following simple Node app, which allows me to call the PetStore endpoint successfully:

const express = require('express');
const app = express();
const crontab = require('node-crontab');
const apiEndpoint = '[snip]';
const APIGatewayClientFactory = require('aws-api-gateway-client').default;

let AWS = require('aws-sdk');
let sts = new AWS.STS();

let params = {};
let AWSCredentials;

let data = {};


function updateEC2CredentialsViaCronjobEveryMinute() {
    crontab.scheduleJob("* * * * *", function() {
        getEC2SessionCredentials()
            .then((credentials) => {
                AWSCredentials = credentials;
            })
            .catch((err) => {
                console.log(err);
            });
    });
}


function getAPIGatewayWrapper(AWSCredentials, params, pathTemplate){
    return APIGatewayClientFactory.newClient({
        invokeUrl: apiEndpoint,
        accessKey: AWSCredentials.Credentials.AccessKeyId,
        secretKey: AWSCredentials.Credentials.SecretAccessKey,
        sessionToken: AWSCredentials.Credentials.SessionToken,
        region: 'eu-west-1',
        apiKey: '[snip]'
    }).invokeApi(params, pathTemplate, 'GET', {}, {})
}


function getEC2SessionCredentials() {
    console.log("getting creds");
    let promise = new Promise((resolve,reject)=>{
        data = sts.getSessionToken(params, function (err, data) {
            if (err)
                reject(err);
            else
                resolve(data);
        })
    });

    return promise;
}


app.get('/', (req, res) => {
    let params = {
        number: '1'
    };

    let pathTemplate = '/pets/{number}';

    getAPIGatewayWrapper(AWSCredentials, params, pathTemplate, res)
    .then((APIGatewayResponse) => {
        data = APIGatewayResponse.data;
        console.log(JSON.stringify(APIGatewayResponse.data.type, null, 4));
        res.status(200).send(data);
    })

    .catch((err)=>{
        console.log(err);
        data = {
                    "error": "an error occurred"
                };
        res.status(500).send(data);
    });

});

getEC2SessionCredentials()
.then((credentials) => {
    AWSCredentials = credentials;
    updateEC2CredentialsViaCronjobEveryMinute();
    app.listen(3000, () => console.log("starting"));
});

I have been experimenting with updating the session AWS credentials (used by the aws-api-gateway-client), and have opted for a Cronjob.

Running with curl gives me the output I am expecting:

http://localhost:3000/?14 --> <stdout>
--_curl_--http://localhost:3000/?14
{"id":1,"type":"dog","price":249.99}

What have I learnt?

The process of implementing my own API gateway, and local Node app has allowed me:

  • To concentrate on the task at hand.

  • Used my own AWS account, my own API keys, and my own IAM roles.

  • Completed a working POC, which has reinforced learning, by staying on track.

  • The code can be further improved, but has allowed me to gain more experience in Node..

What I have avoided?

  • Getting involved in hacking AWS IAM roles.

  • Losing focus.

  • Remote development on an EC2 box.

What I haven’t avoided?

  • Interesting…yes, but spending my own time working on a “work task”

I’ve deliberately gone back to basics over the holidays, kept things simple, and focused, and surprise surprise, things have gone to plan.

»


Serverspec

Serverspec on Centos

Serverspec allow you to develop RSpec aka understandable, repeatable infrastructure tests (

Installing
# yum install rubygems
# gem install bundler
# gem install rake
# gem install serverspec

# serverspec-init
Select OS type:

  1) UN*X
  2) Windows

Select number: 1

Select a backend type:

  1) SSH
  2) Exec (local)

Select number: 2

 + spec/
 + spec/localhost/
 + spec/localhost/sample_spec.rb
 + spec/spec_helper.rb
 + Rakefile
 + .rspec

Editing - the spec file (spec/localhost/sample_spec.rb)

require 'spec_helper'

describe package('httpd'), :if => os[:family] == 'redhat' do
  it { should be_installed }
end

describe package('apache2'), :if => os[:family] == 'ubuntu' do
  it { should be_installed }
end

describe service('httpd'), :if => os[:family] == 'redhat' do
  it { should be_enabled }
  it { should be_running }
end

describe service('apache2'), :if => os[:family] == 'ubuntu' do
  it { should be_enabled }
  it { should be_running }
end

describe service('org.apache.httpd'), :if => os[:family] == 'darwin' do
  it { should be_enabled }
  it { should be_running }
end

describe port(80) do
  it { should be_listening }
end

Running

# rake spec
/usr/bin/ruby -I/usr/local/share/gems/gems/rspec-core-3.7.0/lib:/usr/local/share/gems/gems/rspec-support-3.7.0/lib /usr/local/share/gems/gems/rspec-core-3.7.0/exe/rspec --pattern spec/localhost/\*_spec.rb

Package "httpd"
  should be installed

Service "httpd"
  should be enabled
  should be running

Port "80"
  should be listening

Finished in 0.05114 seconds (files took 0.27387 seconds to load)
4 examples, 0 failures
#

All tests pass, this assumes that httpd is installed, enabled, and running on port 80.

»


My Ideal Job

My idea job would be a combination of Software Engineering 70%, and Operations Engineering 30%.

Miles Davenport

Operations Engineering is designing, implementing, and supporting of the automated infrastructure which your software components rely on.

In other words the repeatable (idempotent **) plumbing your systems rely on.

API development based on small micro-services using Java 8, AWS, and Python, using Centos Linux.

I have an interest in Infosec which I would like to develop further. Working for an SME is my preference.

** idempotent - no additional effect, regardless of the number of times (repeatedly) called.

»