Distributed KV Store with example python program (Using Python-evnconsul)

Example use case of using Consul KV to retrieve Configuration values to have a stateless micro service.

Environment:

- Consul agent running at <consul_ip>.
- Django based micro service.
- Manage DB configuration in settings.py of the micro service.

Steps:

Add database configuration key values to Consul:

^{curl -X PUT -d 'django.db.backends.mysql' http://<consul_ip>:8500/v1/kv/<service_name>/database/engine}
^{curl -X PUT -d '<DB_NAME>' http://<consul_ip>:8500/v1/kv/<service_name>/database/name}
^{curl -X PUT -d '<DB_IP>' http://<consul_ip>:8500/v1/kv/<service_name>/database/ip}
^{curl -X PUT -d 3306 http://<consul_ip>8500/v1/kv/<service_name>/database/port}
^{curl -X PUT -d '<DB_USERNAME>' http://<consul_ip>:8500/v1/kv/<service_name>/database/username}
^{curl -X PUT -d '<DB_PASSWORD>' http://<consul_ip>:8500/v1/kv/<service_name>/database/password}

In settings.py of Django, retrieve the DB configuration values as:

^{import envconsul}

^{ENV_CONSUL = envconsul.EnvConsul(}
^{service_name='<service_name>',}
^{host='<consul_ip>',}
^port=8500,
⁾

^{DB_ENGINE = ENV_CONSUL.get_str('/database/engine')}
^{DB_NAME = ENV_CONSUL.get_str('/database/name')}
^{DB_IP = ENV_CONSUL.get_str('/database/ip')}
^{DB_PORT = ENV_CONSUL.get('/database/port')}
^{DB_USERNAME = ENV_CONSUL.get_str('/database/username')}
^{DB_PASSWORD = ENV_CONSUL.get_str('/database/password')}

^{# Use the values obtained from Consul in DATABASES config of Django.}

^{DATABASES = {}
^{'default': {}
^{'ENGINE': DB_ENGINE,}
^{'NAME': DB_NAME,}
^{'HOST': DB_IP,}
^{'PORT': DB_PORT,}
^{'USER': DB_USERNAME,}
^{'PASSWORD': DB_PASSWORD,}
^},
^}

This way, DB configuration can easily be passed to all the micro services.

Example shown is using python-envconsul to fetch KVs. Need to use better library such as python-consul which provides capabilities to fetch KV as notifications whenever there are changes in the values.

<TODO>

Distributed KV Store with example python program (Using Python-Consul)

this is to experiment notification functionality and configuration reloading.

Distributed KV Store with example Java program (Using cfg4J)

Environment:

- Consul agent running at <consul_ip>.
- Java based micro service.
- Manage DB configuration values in a ".configuration" file of a micro service.

Steps:

Add database configuration key values to Consul:

^{curl -X PUT -d '<DB_NAME>' http://<consul_ip>:8500/v1/kv/<service_name>/database/name}
^{curl -X PUT -d '<DB_IP>' http://<consul_ip>:8500/v1/kv/<service_name>/database/ip}
^{curl -X PUT -d 3306 http://<consul_ip>8500/v1/kv/<service_name>/database/port}
^{curl -X PUT -d '<DB_USERNAME>' http://<consul_ip>:8500/v1/kv/<service_name>/database/username}
^{curl -X PUT -d '<DB_PASSWORD>' http://<consul_ip>:8500/v1/kv/<service_name>/database/password}

Example usage of cfg4j to retrieve the above Key Values from Consul:

^{import org.cfg4j.provider.ConfigurationProvider;}
^{import org.cfg4j.provider.ConfigurationProviderBuilder;}
^{import org.cfg4j.source.ConfigurationSource;}
^{import org.cfg4j.source.consul.ConsulConfigurationSourceBuilder;}
^{import org.cfg4j.source.context.environment.ImmutableEnvironment;}
^{import org.cfg4j.source.reload.ReloadStrategy;}
^{import org.cfg4j.source.reload.strategy.PeriodicalReloadStrategy;}
^{import org.cfg4j.source.compose.MergeConfigurationSource;}
^{import org.cfg4j.source.files.FilesConfigurationSource;}

^{import java.lang.reflect.Field;}
^{import java.util.Collections;}
^{import java.nio.file.Path;}
^{import java.nio.file.Paths;}
^{import java.util.HashMap;}
^{import java.util.concurrent.TimeUnit;}

^{public class Consulcf4j {}

^{public interface ReadDBConfig {}
^{String name();}
^{String ip();}
^{Integer port();}
^{String username();}
^{String password();}
^}

^{public static void main(String... args) throws Exception{}

^{HashMap<String, Object> prev_map = new HashMap<String, Object>();}
^{HashMap<String, Object> curr_map = new HashMap<String, Object>();}

^{prev_map.put("name", "");}
^{prev_map.put("ip", "");}
^{prev_map.put("port", 0);}
^{prev_map.put("username", "");}
^{prev_map.put("password", "");}

^{ConfigurationSource source = new ConsulConfigurationSourceBuilder()}
^{.withHost("localhost")}
^.build();

^{ReloadStrategy reloadStrategy = new PeriodicalReloadStrategy(5, TimeUnit.SECONDS);}

^{//ConfigurationSource localOverrideSource = new FilesConfigurationSource(() -> Collections.singletonList(Paths.get("appliction.properties")));}

^{//MergeConfigurationSource mergeConfigurationSource = new MergeConfigurationSource(localOverrideSource, source);}

^{ConfigurationProvider provider = new ConfigurationProviderBuilder()}
^{.withConfigurationSource(source)}
^{.withEnvironment(new ImmutableEnvironment("vfcnslcm"))}
^{.withReloadStrategy(reloadStrategy)}
^.build();

^{ReadDBConfig conf = provider.bind("database", ReadDBConfig.class);}

^while(true){
^{curr_map.put("name", conf.name());}
^{curr_map.put("ip", conf.ip());}
^{curr_map.put("port", conf.port());}
^{curr_map.put("username", conf.username());}
^{curr_map.put("password", conf.password());}

^{for(String key: curr_map.keySet()){}
^{if(!prev_map.get(key).equals(curr_map.get(key))){}
^{System.out.println("Values: ");}
^{for(String k: curr_map.keySet()){}
^{System.out.println(curr_map.get(k));}
^}
^break;
^}
^}

^{prev_map.clear();}
^{prev_map.putAll(curr_map);}

^}

^{// String value = provider.getProperty("engine", String.class);}
^{// System.out.println(value);}
^}
^}

This way, DB configuration can easily be passed to all the micro services.

This can be used in Spring Beans as shown in http://www.cfg4j.org/releases/latest/#configuration-provider

Distributed KV Store with one of ONAP Python Service

Distributed KV Store with one of ONAP Java Service

ONAP Wiki

Distributed KV Store - PoC / Experiements

Distributed KV Store with example python program (Using Python-evnconsul)

Distributed KV Store with example python program (Using Python-Consul)

Distributed KV Store with example Java program (Using cfg4J)

Distributed KV Store with one of ONAP Python Service

Distributed KV Store with one of ONAP Java Service