Skip to content

Commit

Permalink
Merge pull request #7 in CORE/java-vtl from feature/KP-2147-forbindel…
Browse files Browse the repository at this point in the history 
…se-til-klass to develop

* commit 'df1008f1336d21cbb1d131295bba156a642bf5ea':
  KP-2147 Adds comment on validTo
  KP-2147 Make it possible to accept more parameters when querying KLASS
  KP-2147 Make data structure static
  KP-2147 Make class static
  KP-2147 Throw correct exception
  KP-2147 Removes unused converter
  KP-2147 Add new connector to console
  KP-2147 Add test for canHandle
  KP-2147 Fixes test: join must have same IC names
  Use rename to avoid error in testCheckSingleRuleWithJoin()
  KP-2147 Adds connector for KLASS
  • Loading branch information
@pawbu
pawbu committed Feb 28, 2017
2 parents 33e6fdb + df1008f commit 13b0c13
Show file tree
Hide file tree
Showing 5 changed files with 4,349 additions and 5 deletions.
254 changes: 254 additions & 0 deletions java-vtl-ssb-api-connector/src/main/java/no/ssb/vtl/connectors/SsbKlassApiConnector.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,254 @@
package no.ssb.vtl.connectors;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.datatype.guava.GuavaModule;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import no.ssb.vtl.connector.Connector;
import no.ssb.vtl.connector.ConnectorException;
import no.ssb.vtl.connector.NotFoundException;
import no.ssb.vtl.model.Component;
import no.ssb.vtl.model.DataStructure;
import no.ssb.vtl.model.Dataset;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriTemplate;

import java.io.IOException;
import java.lang.String;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;

import static com.google.common.base.Preconditions.*;
import static java.lang.String.format;
import static java.util.Arrays.*;


/**
* A VTL connector that gets data from KLASS part of api.ssb.no.
*/
public class SsbKlassApiConnector implements Connector {

private static final String SERVICE_URL = "http://data.ssb.no/api/klass/v1";
private static final String[] DATA_PATHS = new String[]{
"/classifications/{classificationId}/codes?from={codesFrom}"
//for example "/classifications/{classificationId}/codes?from={codesFrom}&to={codesTo}"
};
private static final String KLASS_DATE_PATTERN = "yyyy-MM-dd";
private static final String FIELD_CODE = "code";
private static final String FIELD_VALID_FROM = "validFrom";
private static final String FIELD_VALID_TO = "validTo";
private static final String FIELD_NAME = "name";

private static final DataStructure DATA_STRUCTURE =
DataStructure.builder()
.put(FIELD_CODE, Component.Role.IDENTIFIER, String.class)
.put(FIELD_VALID_FROM, Component.Role.IDENTIFIER, Instant.class)
//Note: validTo can contain nulls and VTL specification states that ICs cannot contain null values (VTL 1.1, user manual, line 2283).
//Nevertheless we set validTo to be an Identifier as we're not sure at this point what implications we
//could come upon.
.put(FIELD_VALID_TO, Component.Role.IDENTIFIER, Instant.class)
.put(FIELD_NAME, Component.Role.MEASURE, String.class)
.build();

private final List<UriTemplate> dataTemplates;
private final ObjectMapper mapper;
private final RestTemplate restTemplate;

/*
The list of available datasets:
http://data.ssb.no/api/klass/v1/classifications/search?query=kommune
Example dataset:
http://data.ssb.no/api/klass/v1/classifications/131/codes?from=2016-01-01
*/

public SsbKlassApiConnector(ObjectMapper mapper) {

this.dataTemplates = Lists.newArrayList();
for (String path : DATA_PATHS) {
this.dataTemplates.add(new UriTemplate(SERVICE_URL + path));
}

this.mapper = checkNotNull(mapper, "the mapper was null").copy();

this.mapper.configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false);
this.mapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);

this.mapper.registerModule(new GuavaModule());
this.mapper.registerModule(new Jdk8Module());
this.mapper.registerModule(new JavaTimeModule());

SimpleModule module = new SimpleModule();
module.addDeserializer(Map.class, new KlassDeserializer());
this.mapper.registerModule(module);

MappingJackson2HttpMessageConverter jacksonConverter;
jacksonConverter = new MappingJackson2HttpMessageConverter(this.mapper);

this.restTemplate = new RestTemplate(asList(
jacksonConverter
));

}

/**
* Gives access to the rest template to tests.
*/
RestTemplate getRestTemplate() {
return restTemplate;
}

public boolean canHandle(String url) {
UriTemplate matchFound = findFirstMatchingUriTemplate(url).orElse(null);
return matchFound != null;
}

private Optional<UriTemplate> findFirstMatchingUriTemplate(String url) {
return dataTemplates.stream()
.filter(t -> t.matches(url))
.findFirst();
}

public Dataset getDataset(String url) throws ConnectorException {

try {
HttpHeaders headers = new HttpHeaders();
headers.setAccept(Arrays.asList(MediaType.APPLICATION_JSON));
HttpEntity<String> entity = new HttpEntity<>("parameters", headers);

//http://data.ssb.no/api/klass/v1/classifications/131/codes?from=2016-01-01
ResponseEntity<DatasetWrapper> exchange = restTemplate.exchange(
url,
HttpMethod.GET,
entity, DatasetWrapper.class);

if (exchange.getBody() == null || exchange.getBody().getCodes().size() == 0) {
throw new NotFoundException(format("empty dataset returned for the identifier %s", url));
}

List<Map<String, Object>> datasets = exchange.getBody().getCodes();

return new Dataset() {
@Override
public DataStructure getDataStructure() {
return DATA_STRUCTURE;
}

@Override
public Stream<Tuple> get() {
DataStructure dataStructure = getDataStructure();
Set<String> codeFields = dataStructure.keySet();
return datasets.stream()
.map(d -> Maps.filterKeys(d, codeFields::contains))
.map(d -> convertType(d))
.map(dataStructure::wrap);
}
};

} catch (RestClientException rce) {
throw new ConnectorException(
format("error when accessing the dataset with ids %s", url),
rce
);
}
}

private Map<String, Object> convertType(Map<String, Object> d) {
Map<String, Object> copy = Maps.newLinkedHashMap(d);
Map<String, Class<?>> types = DATA_STRUCTURE.getTypes();

d.forEach((k, v) -> copy.put(k, mapper.convertValue(v, types.get(k))));

return copy;
}

public Dataset putDataset(String identifier, Dataset dataset) throws ConnectorException {
throw new ConnectorException("not supported");
}

static class DatasetWrapper {
@JsonProperty
private List<Map<String, Object>> codes;

public DatasetWrapper() {
}

public List<Map<String, Object>> getCodes() {
return codes;
}

public void setCodes(List<Map<String, Object>> codes) {
this.codes = codes;
}
}

private static class KlassDeserializer extends StdDeserializer<Map<String, Object>> {

public KlassDeserializer() {
this(null);
}

public KlassDeserializer(Class<?> vc) {
super(vc);
}

@Override
public Map<String, Object> deserialize(JsonParser jp, DeserializationContext ctxt)
throws IOException {
JsonNode node = jp.getCodec().readTree(jp);
String code = node.get(FIELD_CODE).asText();
String name = node.get(FIELD_NAME).asText();
String validFromAsString = node.get(FIELD_VALID_FROM).asText();
String validToAsString = node.get(FIELD_VALID_TO).asText();

HashMap<String, Object> entry = Maps.newHashMap();
entry.put(FIELD_CODE, code);
entry.put(FIELD_NAME, name);
entry.put(FIELD_VALID_FROM, parseKlassDate(validFromAsString));
entry.put(FIELD_VALID_TO, parseKlassDate(validToAsString));

return entry;
}
}

public static Instant parseKlassDate(String input) throws JsonParseException {
SimpleDateFormat dateFormat = new SimpleDateFormat(KLASS_DATE_PATTERN);
if (input != null && !input.isEmpty() && !input.toLowerCase().equals("null")) {
try {
Date date = dateFormat.parse(input);
return (date != null) ? date.toInstant() : null;
} catch (ParseException e) {
throw new JsonParseException(null, "Could not parse input to date. Data: "
+ input + ", required format: " + KLASS_DATE_PATTERN);
}
}
return null;
}
}
90 changes: 90 additions & 0 deletions java-vtl-ssb-api-connector/src/test/java/no/ssb/vtl/connectors/SsbKlassApiConnectorTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package no.ssb.vtl.connectors;

import static org.assertj.core.api.Assertions.*;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.*;
import static org.springframework.test.web.client.response.MockRestResponseCreators.*;

import java.io.InputStream;
import java.time.Instant;
import java.time.OffsetDateTime;

import org.junit.Before;
import org.junit.Test;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.test.web.client.MockRestServiceServer;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.io.Resources;

import no.ssb.vtl.connector.Connector;
import no.ssb.vtl.model.Component;
import no.ssb.vtl.model.DataPoint;
import no.ssb.vtl.model.Dataset;

public class SsbKlassApiConnectorTest {

private ObjectMapper mapper;
private Connector connector;
private MockRestServiceServer mockServer;

@Before
public void setUp() throws Exception {
this.mapper = new ObjectMapper();
SsbKlassApiConnector ssbConnector = new SsbKlassApiConnector(this.mapper);
this.connector = ssbConnector;
mockServer = MockRestServiceServer.createServer(ssbConnector.getRestTemplate());
}


@Test
public void testCanHandle() throws Exception {

String testUri = "http://data.ssb.no/api/klass/v1/classifications/131/codes?from=2013-01-01";
assertThat(this.connector.canHandle(testUri));

testUri = "http://data.ssb.no/api/v0/dataset/1106.json?lang=en";
assertThat(!this.connector.canHandle(testUri));

}

@Test
public void testGetDataset() throws Exception {

InputStream fileStream = Resources.getResource(this.getClass(), "/codes131_from2013.json").openStream();

mockServer.expect(requestTo("http://data.ssb.no/api/klass/v1/classifications/131/codes?from=2013-01-01"))
.andExpect(method(HttpMethod.GET))
.andRespond(withSuccess(
new InputStreamResource(fileStream),
MediaType.APPLICATION_JSON)
);

Dataset dataset = this.connector.getDataset("http://data.ssb.no/api/klass/v1/classifications/131/codes?from=2013-01-01");

assertThat(dataset.getDataStructure().getRoles()).containsOnly(
entry("code", Component.Role.IDENTIFIER),
entry("name", Component.Role.MEASURE),
entry("validFrom", Component.Role.IDENTIFIER),
entry("validTo", Component.Role.IDENTIFIER)
);

assertThat(dataset.getDataStructure().getTypes()).containsOnly(
entry("code", String.class),
entry("name", String.class),
entry("validFrom", Instant.class),
entry("validTo", Instant.class)
);

assertThat(dataset.get())
.flatExtracting(input -> input)
.extracting(DataPoint::get)
.containsSequence(
"0101", "Halden", OffsetDateTime.parse("2012-12-31T23:00:00Z").toInstant(), null,
"0104", "Moss", OffsetDateTime.parse("2012-12-31T23:00:00Z").toInstant(), null
);

}

}
Loading

0 comments on commit 13b0c13

Please sign in to comment.