Add weather processing

.gitignore

@@ -1 +1,2 @@
 .vscode
+weather.db

readme.md

@@ -1,8 +1,27 @@
-# Ringier Silly Aggregator
+# Ringier Weather Aggregator
+
+This is an example project created to help with technical interviews.
 
 ## Getting Started
 
-This project is a simple HTTP server with routing capabilities. Follow the instructions below to run the server and execute tests.
+Follow these steps to get a general idea of what this app is about:
+
+1. make sure you have deno installed
+1. run the server
+2. navigate to `/sync-weather`
+3. wait a bit for some data to import
+4. navigate to `/` to see the graph
+
+### Installing Deno
+
+On Mac, run:
+
+```
+brew install deno
+```
+
+For instructions installing homebrew, go to <https://brew.sh>
+For instructions installed Deno, do to <https://deno.com>
 
 ### Running the Server
 

responders/index-responder.ts

@@ -0,0 +1,30 @@
+import { connect, view } from "../server-helpers.ts";
+
+export default async () => {
+    const query = `
+        SELECT
+            DATE(date) AS day,
+            MIN(temperature) AS minimumTemperature,
+            MAX(temperature) AS maximumTemperature
+        FROM
+            weather
+        GROUP BY
+            DATE(date)
+        ORDER BY
+            day;
+    `;
+
+    const db = connect();
+
+    const results = db.query(query);
+
+    const data = results
+        .map(([day, minimumTemperature, maximumTemperature]) => ({
+            day,
+            minimumTemperature: parseFloat(minimumTemperature as string),
+            maximumTemperature: parseFloat(maximumTemperature as string),
+        }))
+        .map(item => JSON.stringify(item));
+
+    return await view('index', { data });
+}

responders/sync-weather-responder.ts

@@ -0,0 +1,61 @@
+import { parse } from "jsr:@std/csv";
+import { connect, insertWeatherData, marshalWeatherData } from "../server-helpers.ts";
+import { createWeatherTable } from "../server-helpers.ts";
+
+export default () => {
+    downloadAndProcess().catch(console.error);
+
+    return new Response("Weather", { status: 200 });
+}
+
+let columns: string[] = [];
+
+const processLine = (line: string) => {
+
+    if (columns.length === 0) {
+        columns = line.trim().split(",");
+        return;
+    }
+
+    const data = parse(line, {
+        columns,
+    });
+
+    const marshalled = marshalWeatherData(data[0]);
+
+    insertWeatherData(connect(), marshalled);
+};
+
+const downloadAndProcess = async () => {
+    const response = await fetch("https://assertchris.fra1.cdn.digitaloceanspaces.com/ad-hoc/14-08-2024-ringier-silly-aggregator/Cape_Town_-33_922087_18_423142_66bc2723bf8a6c0008de9189.csv");
+
+    if (!response.body) {
+        throw new Error("Failed to fetch file");
+    }
+
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder("utf-8");
+    let buffer = "";
+
+    while (true) {
+        const { done, value } = await reader.read();
+
+        if (done) {
+            break;
+        }
+
+        buffer += decoder.decode(value, { stream: true });
+
+        let newlineIndex;
+
+        while ((newlineIndex = buffer.indexOf("\n")) >= 0) {
+            const line = buffer.slice(0, newlineIndex);
+            buffer = buffer.slice(newlineIndex + 1);
+            processLine(line);
+        }
+    }
+
+    if (buffer.length > 0) {
+        processLine(buffer);
+    }
+};

routes.ts

@@ -1,3 +1,6 @@
+import indexResponder from "./responders/index-responder.ts";
+import syncWeatherResponder from "./responders/sync-weather-responder.ts";
+
 export type Route = {
     method: string;
     pattern: string; // Route pattern with placeholders (e.g., "/user/:id")
@@ -8,6 +11,11 @@ export const routes: Route[] = [
     {
         method: "GET",
         pattern: "/",
-        handler: () => new Response("Welcome to the homepage!", { status: 200 }),
+        handler: indexResponder,
     },
+    {
+        method: "GET",
+        pattern: "/sync-weather",
+        handler: syncWeatherResponder,
+    }
 ];

server-helpers.ts

@@ -1,3 +1,6 @@
+import { DB } from "https://deno.land/x/[email protected]/mod.ts";
+import { Eta } from "https://deno.land/x/[email protected]/src/index.ts"
+
 export type RouteMatch = {
     matched: boolean;
     params: Record<string, string>;
@@ -23,3 +26,138 @@ export const matchRoute = (pattern: string, url: string): RouteMatch => {
 
     return { matched: true, params };
 };
+
+export type WeatherData = {
+    date: Date;
+    city: string;
+    latitude: string;
+    longitude: string;
+    temperature: number;
+    visibility: string;
+    dewPoint: number;
+    feelsLike: number;
+    minimumTemperature: number;
+    maximumTemperature: number;
+    pressure: string;
+    seaLevel: string;
+    groundLevel: string;
+    humidity: number;
+    windSpeed: number;
+    windDirection: number;
+    windGust: string;
+    weatherDescription: string;
+}
+
+export const kelvinToCelsius = (kelvin: string): number => {
+    const kelvinValue = parseFloat(kelvin);
+    return kelvinValue - 273.15;
+};
+
+export const marshalWeatherData = (data: Record<string, string>): WeatherData => {
+    return {
+        date: new Date(data.dt_iso),
+        city: String(data.city_name),
+        latitude: String(data.lat),
+        longitude: String(data.lon),
+        temperature: kelvinToCelsius(data.temp),
+        visibility: String(data.visibility),
+        dewPoint: kelvinToCelsius(data.dew_point),
+        feelsLike: kelvinToCelsius(data.feels_like),
+        minimumTemperature: kelvinToCelsius(data.temp_min),
+        maximumTemperature: kelvinToCelsius(data.temp_max),
+        pressure: String(data.pressure),
+        seaLevel: String(data.sea_level),
+        groundLevel: String(data.grnd_level),
+        humidity: parseFloat(String(data.humidity)),
+        windSpeed: parseFloat(String(data.wind_speed)),
+        windDirection: parseFloat(String(data.wind_deg)),
+        windGust: String(data.wind_gust),
+        weatherDescription: String(data.weather_description),
+    };
+};
+
+export const connect = () => {
+    return new DB("weather.db");
+};
+
+export const createWeatherTable = (db: DB) => {
+    db.execute(`
+        CREATE TABLE IF NOT EXISTS weather (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            date TEXT NOT NULL,
+            city TEXT NOT NULL,
+            latitude TEXT NOT NULL,
+            longitude TEXT NOT NULL,
+            temperature REAL NOT NULL,
+            visibility TEXT,
+            dewPoint REAL NOT NULL,
+            feelsLike REAL NOT NULL,
+            minimumTemperature REAL NOT NULL,
+            maximumTemperature REAL NOT NULL,
+            pressure TEXT NOT NULL,
+            seaLevel TEXT,
+            groundLevel TEXT,
+            humidity INTEGER NOT NULL,
+            windSpeed REAL NOT NULL,
+            windDirection INTEGER NOT NULL,
+            windGust TEXT,
+            weatherDescription TEXT NOT NULL
+        );
+    `);
+};
+
+export const insertWeatherData = (db: DB, weatherData: WeatherData) => {
+    createWeatherTable(db);
+
+    const checkQuery = `SELECT COUNT(*) FROM weather WHERE date = ?`;
+    const result = db.query(checkQuery, [weatherData.date.toISOString()]);
+    const [count] = result[0] as number[];
+
+    if (count > 0) {
+        // console.log(`Data for date ${weatherData.date.toISOString()} already exists.`);
+        return;
+    }
+
+    const query = `
+      INSERT INTO weather (
+        date, city, latitude, longitude, temperature, visibility, dewPoint,
+        feelsLike, minimumTemperature, maximumTemperature, pressure, seaLevel,
+        groundLevel, humidity, windSpeed, windDirection, windGust, weatherDescription
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `;
+
+    db.query(query, [
+      weatherData.date.toISOString(),
+      weatherData.city,
+      weatherData.latitude,
+      weatherData.longitude,
+      weatherData.temperature,
+      weatherData.visibility,
+      weatherData.dewPoint,
+      weatherData.feelsLike,
+      weatherData.minimumTemperature,
+      weatherData.maximumTemperature,
+      weatherData.pressure,
+      weatherData.seaLevel,
+      weatherData.groundLevel,
+      weatherData.humidity,
+      weatherData.windSpeed,
+      weatherData.windDirection,
+      weatherData.windGust,
+      weatherData.weatherDescription
+    ]);
+};
+
+export const view = async function(name: string, data: object = {}): Promise<Response> {
+    const renderer = new Eta({
+        views: "./templates/",
+        cache: true,
+        useWith: true,
+    });
+
+    const content = await renderer.renderAsync(name, data);
+
+    return new Response(content, {
+        headers: { "Content-Type": "text/html" }
+    })
+};

templates/index.eta

@@ -0,0 +1,90 @@
+<style>
+    svg {
+        font-family: Arial, sans-serif;
+    }
+    .axis {
+        stroke: #000;
+    }
+    .line {
+        stroke-width: 2;
+        fill: none;
+    }
+    .dot {
+        stroke: #1f77b4;
+        fill: #1f77b4;
+    }
+</style>
+<div id="chart"></div>
+<script>
+    function formatMonthYear(dateString) {
+        const [year, month] = dateString.split('-').map(Number);
+        const monthNames = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+        return `${monthNames[month - 1]} ${year}`;
+    }
+
+    function aggregateData(data) {
+        const groupedData = {};
+
+        data.forEach(d => {
+            const date = new Date(d.day);
+            const key = formatMonthYear(`${date.getFullYear()}-${date.getMonth() + 1}`);
+
+            if (!groupedData[key]) {
+                groupedData[key] = { minTempSum: 0, maxTempSum: 0, count: 0 };
+            }
+            groupedData[key].minTempSum += d.minimumTemperature;
+            groupedData[key].maxTempSum += d.maximumTemperature;
+            groupedData[key].count += 1;
+        });
+
+        return Object.entries(groupedData).map(([key, value]) => ({
+            period: key,
+            avgMinTemp: value.minTempSum / value.count,
+            avgMaxTemp: value.maxTempSum / value.count
+        }));
+    }
+
+    const data = aggregateData([<%~ data %>]);
+
+    const width = 800;
+    const height = 400;
+    const barWidth = width / data.length - 14;
+    const margin = { top: 20, right: 20, bottom: 80, left: 60 };
+    const innerHeight = height - margin.top - margin.bottom;
+
+    const maxTemp = Math.max(...data.map(d => d.avgMaxTemp));
+    const yScale = innerHeight / maxTemp;
+
+    let svgContent = `<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">`;
+
+    const yAxisStep = 3;
+
+    for (let i = 0; i <= maxTemp; i += yAxisStep) {
+        const y = height - margin.bottom - i * yScale;
+        svgContent += `<line x1="${margin.left}" y1="${y}" x2="${width - margin.right}" y2="${y}" stroke="#ccc" stroke-width="0.5"/>`;
+        svgContent += `<text x="${margin.left - 5}" y="${y + 4}" fill="black" font-size="12" text-anchor="end">${i}&deg;C</text>`;
+    }
+
+    data.forEach((d, index) => {
+        const x = margin.left + index * (barWidth + 10);
+        const yMax = height - margin.bottom - d.avgMaxTemp * yScale;
+        const barHeightMax = d.avgMaxTemp * yScale;
+
+        const yMin = height - margin.bottom - d.avgMinTemp * yScale;
+        const barHeightMin = d.avgMinTemp * yScale;
+
+        svgContent += `<rect x="${x}" y="${yMax}" width="${barWidth}" height="${barHeightMax}" fill="blue" />`;
+        svgContent += `<rect x="${x}" y="${yMin}" width="${barWidth}" height="${barHeightMin}" fill="lightblue" />`;
+
+        svgContent += `<text x="${x + barWidth / 2}" y="${height - margin.bottom + 40}" fill="black" font-size="12" text-anchor="middle" transform="rotate(-45 ${x + barWidth / 2},${height - margin.bottom + 40})">${d.period}</text>`;
+    });
+
+    svgContent += `<line x1="${margin.left}" y1="${height - margin.bottom}" x2="${width - margin.right}" y2="${height - margin.bottom}" stroke="black" />`;
+    svgContent += `<line x1="${margin.left}" y1="${margin.top}" x2="${margin.left}" y2="${height - margin.bottom}" stroke="black" />`;
+
+    svgContent += `<text x="${margin.left - 40}" y="${(height / 2) - 6}" fill="black" font-size="14" text-anchor="middle" transform="rotate(-90 ${margin.left - 40},${height / 2})">Temperature (&deg;C)</text>`;
+
+    svgContent += '</svg>';
+
+    document.getElementById('chart').innerHTML = svgContent;
+</script>